{ "cells": [ { "cell_type": "markdown", "id": "133a67f9", "metadata": {}, "source": [ "# Crowd SDK MCP — Statistic Demo\n", "\n", "This notebook is designed for a live demo of the statistic module tools exposed by the local Crowd SDK MCP server.\n", "\n", "## Demo flow\n", "\n", "1. Show how the MCP server is connected.\n", "2. Show which tools are exposed.\n", "3. Run `health_check` and `get_platform_info`.\n", "4. Run statistic module tools: `get_project_stats`, `get_task_stats`, `get_annotation_summary`, `get_marker_performance_stats`, `analyze_task_results`, `analyze_fraud_risk`, `analyze_project_results`, `analyze_project_instructions`.\n", "5. Show how `get_annotation_summary` returns an LLM-friendly response shape.\n", "6. Show an end-to-end flow where an LLM decides to call an MCP tool and uses the result in the final answer.\n", "\n", "## MCP config example\n", "\n", "```json\n", "{\n", " \"mcpServers\": {\n", " \"crowd-sdk\": {\n", " \"command\": \"tagme\",\n", " \"args\": [\"mcp\"]\n", " }\n", " }\n", "}\n", "```\n", "\n", "For this notebook, we connect to the same server directly over stdio." ] }, { "cell_type": "code", "execution_count": null, "id": "c5ec7ecd", "metadata": {}, "outputs": [], "source": [ "import json\n", "import os\n", "from pathlib import Path\n", "from typing import Any, Dict\n", "from urllib import error, request\n", "\n", "import ssl\n", "\n", "ssl._create_default_https_context = ssl._create_unverified_context\n", "\n", "from mcp import ClientSession, StdioServerParameters\n", "from mcp.client.stdio import stdio_client\n", "\n", "ENV_FILE = Path('/path/to/.env')\n", "MCP_COMMAND = 'tagme'\n", "MCP_ARGS = ['mcp']" ] }, { "cell_type": "code", "execution_count": 19, "id": "703f52b0", "metadata": {}, "outputs": [], "source": [ "def load_env(path: Path) -> None:\n", " if not path.exists():\n", " return\n", "\n", " for raw_line in path.read_text(encoding='utf-8').splitlines():\n", " line = raw_line.strip()\n", " if not line or line.startswith('#') or '=' not in line:\n", " continue\n", "\n", " key, value = line.split('=', 1)\n", " os.environ.setdefault(key.strip(), value.strip().strip('\"').strip(\"'\"))\n", "\n", "\n", "def server_params() -> StdioServerParameters:\n", " load_env(ENV_FILE)\n", " return StdioServerParameters(\n", " command=MCP_COMMAND,\n", " args=MCP_ARGS,\n", " env=os.environ.copy(),\n", " cwd=str(Path.cwd()),\n", " )\n", "\n", "\n", "def payload_from_mcp_result(tool_result: Any) -> Any:\n", " structured_content = getattr(tool_result, 'structuredContent', None) or {}\n", " if 'result' in structured_content:\n", " return structured_content['result']\n", " return [content.model_dump(mode='json') for content in tool_result.content]\n", "\n", "\n", "async def list_tools() -> list[str]:\n", " async with stdio_client(server_params()) as (read, write):\n", " async with ClientSession(read, write) as session:\n", " await session.initialize()\n", " response = await session.list_tools()\n", " return sorted(tool.name for tool in response.tools)\n", "\n", "\n", "async def list_openai_tools() -> list[Dict[str, Any]]:\n", " async with stdio_client(server_params()) as (read, write):\n", " async with ClientSession(read, write) as session:\n", " await session.initialize()\n", " response = await session.list_tools()\n", " return [\n", " {\n", " 'type': 'function',\n", " 'function': {\n", " 'name': tool.name,\n", " 'description': tool.description or f'MCP tool: {tool.name}',\n", " 'parameters': getattr(tool, 'inputSchema', None) or {'type': 'object', 'properties': {}},\n", " },\n", " }\n", " for tool in response.tools\n", " ]\n", "\n", "\n", "async def call_tool(tool_name: str, arguments: Dict[str, Any] | None = None) -> Any:\n", " async with stdio_client(server_params()) as (read, write):\n", " async with ClientSession(read, write) as session:\n", " await session.initialize()\n", " result = await session.call_tool(tool_name, arguments or {})\n", " return payload_from_mcp_result(result)\n", "\n", "\n", "def show(title: str, payload: Any) -> None:\n", " print(f'\\n=== {title} ===')\n", " print(json.dumps(payload, indent=2, ensure_ascii=False))\n", "\n", "\n", "def openai_chat_completion(\n", " messages: list[Dict[str, Any]],\n", " tools: list[Dict[str, Any]],\n", " model: str,\n", " api_key: str,\n", " base_url: str,\n", ") -> Dict[str, Any]:\n", " payload = {\n", " 'model': model,\n", " 'messages': messages,\n", " 'tools': tools,\n", " 'tool_choice': 'auto',\n", " 'temperature': 0,\n", " }\n", " req = request.Request(\n", " url=f'{base_url}/chat/completions',\n", " data=json.dumps(payload).encode('utf-8'),\n", " headers={\n", " 'Authorization': f'Bearer {api_key}',\n", " 'Content-Type': 'application/json',\n", " },\n", " method='POST',\n", " )\n", "\n", " try:\n", " with request.urlopen(req) as response:\n", " return json.loads(response.read().decode('utf-8'))\n", " except error.HTTPError as exc:\n", " details = exc.read().decode('utf-8', errors='replace')\n", " raise RuntimeError(f'LLM request failed with status {exc.code}: {details}') from exc" ] }, { "cell_type": "markdown", "id": "3098a422", "metadata": {}, "source": [ "## 1. Quick smoke demo\n", "\n", "This is the fastest way to show that the MCP server is alive and exposes the expected tool surface." ] }, { "cell_type": "code", "execution_count": 20, "id": "3e84a334", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== available_tools ===\n", "[\n", " \"create_case_mock\",\n", " \"get_annotation_summary\",\n", " \"get_marker_performance_stats\",\n", " \"get_platform_info\",\n", " \"get_project_quality_stats\",\n", " \"get_project_stats\",\n", " \"get_task_stats\",\n", " \"health_check\",\n", " \"list_cases_mock\"\n", "]\n", "\n", "=== health_check ===\n", "{\n", " \"status\": \"ok\",\n", " \"service\": \"crowd-sdk-mcp\"\n", "}\n", "\n", "=== get_platform_info ===\n", "{\n", " \"name\": \"Crowd SDK\",\n", " \"version\": \"4.10.0\",\n", " \"mcp_enabled\": true\n", "}\n" ] } ], "source": [ "tools = await list_tools()\n", "show('available_tools', tools)\n", "\n", "health = await call_tool('health_check')\n", "show('health_check', health)\n", "\n", "platform_info = await call_tool('get_platform_info')\n", "show('get_platform_info', platform_info)" ] }, { "cell_type": "code", "execution_count": null, "id": "1b2d0367", "metadata": {}, "outputs": [], "source": [ "PROJECT_ID = \"180f3050-7a75-...\" # replace with a real project id for the live demo\n", "TASK_ID = \"21fa9026-7482-...\" # replace with a real task id for the live demo\n", "ORGANIZATION_ID = \"8f9375d6-4f3a-48d5-...\"\n", "FROM_DATE = None # format: YYYY-MM-DD\n", "TO_DATE = None # format: YYYY-MM-DD\n", "INCLUDE_QUALITY = False\n", "QUALITY_LIMIT = 10" ] }, { "cell_type": "markdown", "id": "c42d8e05", "metadata": {}, "source": [ "## 2. Live project demo\n", "\n", "Set `PROJECT_ID` above before running this cell. Optionally enable `INCLUDE_QUALITY` and date range." ] }, { "cell_type": "code", "execution_count": null, "id": "74bd5ee9", "metadata": {}, "outputs": [], "source": [ "if not PROJECT_ID:\n", " print('Set PROJECT_ID in the setup cell before running the live project demo.')\n", "else:\n", " project_args = {'project_id': PROJECT_ID}\n", " if ORGANIZATION_ID:\n", " project_args['organization_id'] = ORGANIZATION_ID\n", "\n", " project_stats = await call_tool('get_project_stats', project_args)\n", " show('get_project_stats', project_stats)\n", "\n", " summary_args = dict(project_args)\n", " summary_args['include_quality'] = INCLUDE_QUALITY\n", " if FROM_DATE and TO_DATE:\n", " summary_args['from_date'] = FROM_DATE\n", " summary_args['to_date'] = TO_DATE\n", "\n", " project_summary = await call_tool('get_annotation_summary', summary_args)\n", " show('get_annotation_summary(project)', project_summary)\n", "\n", " if INCLUDE_QUALITY:\n", " quality_args = {\n", " 'project_ids': [PROJECT_ID],\n", " 'limit': QUALITY_LIMIT,\n", " }\n", " if ORGANIZATION_ID:\n", " quality_args['organization_id'] = ORGANIZATION_ID\n", " project_quality = await call_tool('get_project_quality_stats', quality_args)\n", " show('get_project_quality_stats(project)', project_quality)" ] }, { "cell_type": "markdown", "id": "be351f43", "metadata": {}, "source": [ "## 3. Live task demo\n", "\n", "Set `TASK_ID` above before running this cell." ] }, { "cell_type": "code", "execution_count": 6, "id": "5519d0a2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== get_task_stats ===\n", "{\n", " \"entity_type\": \"task\",\n", " \"task\": {\n", " \"uid\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"name\": \"🤖 NLAP TEXT | Разметка DLAP\",\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"created_date\": \"2026-04-15T18:45:32.130250+03:00\",\n", " \"update_date\": \"2026-04-22T15:51:42.940953+03:00\",\n", " \"start_date\": \"2026-04-16T13:28:06.160329+03:00\",\n", " \"finished_date\": \"2026-04-22T15:51:41.060853+03:00\",\n", " \"state\": \"DONE\",\n", " \"overlap\": 3,\n", " \"price\": 8.0,\n", " \"estimated_files_count\": null,\n", " \"markers_count_plan\": null\n", " },\n", " \"snapshot\": {\n", " \"task_id\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"total_marked_time\": 160905.7,\n", " \"objects_count\": 550,\n", " \"marked_count\": 500,\n", " \"total_skipped_objects_count\": 0,\n", " \"total_marked_objects_count\": 1777,\n", " \"total_accepted_objects_count\": 1777,\n", " \"total_rejected_objects_count\": 0,\n", " \"current_overlap\": 3,\n", " \"total_objects_count\": 1500,\n", " \"avg_object_count_marker\": 126.93,\n", " \"status\": \"DONE\",\n", " \"one_object_avg_time\": 90.55,\n", " \"active_markers\": 0,\n", " \"total_markers\": 14,\n", " \"finished_marker\": 0,\n", " \"pretending_marker\": 0,\n", " \"cost\": 14216.0,\n", " \"avg_quality\": 0.79,\n", " \"avg_consistency\": 0.8,\n", " \"avg_hour_profit\": 565186.09\n", " },\n", " \"progress\": {\n", " \"marked_count\": 500,\n", " \"objects_count\": 550,\n", " \"progress_percent\": 90.91\n", " },\n", " \"workload\": {\n", " \"current_overlap\": 3,\n", " \"active_markers\": 0,\n", " \"total_markers\": 14,\n", " \"avg_object_count_marker\": 126.93,\n", " \"one_object_avg_time\": 90.55,\n", " \"total_marked_time\": 160905.7\n", " },\n", " \"quality\": {\n", " \"average_quality\": 0.79,\n", " \"average_consistency\": 0.8,\n", " \"average_hour_profit\": 565186.09\n", " },\n", " \"status_breakdown\": {\n", " \"accepted_objects\": 1777,\n", " \"rejected_objects\": 0,\n", " \"marked_objects\": 1777,\n", " \"skipped_objects\": 0\n", " },\n", " \"summary\": \"Task \\\"🤖 NLAP TEXT | Разметка DLAP\\\" state=DONE, marked=500/550, progress=90.91%, overlap=3, active_markers=0, avg_quality=0.79, avg_consistency=0.8\",\n", " \"advanced\": {\n", " \"estimation_date\": null,\n", " \"in_progress_count\": 0\n", " }\n", "}\n", "\n", "=== get_annotation_summary(task) ===\n", "{\n", " \"entity_type\": \"task\",\n", " \"task\": {\n", " \"uid\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"name\": \"🤖 NLAP TEXT | Разметка DLAP\",\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"created_date\": \"2026-04-15T18:45:32.130250+03:00\",\n", " \"update_date\": \"2026-04-22T15:51:42.940953+03:00\",\n", " \"start_date\": \"2026-04-16T13:28:06.160329+03:00\",\n", " \"finished_date\": \"2026-04-22T15:51:41.060853+03:00\",\n", " \"state\": \"DONE\",\n", " \"overlap\": 3,\n", " \"price\": 8.0,\n", " \"estimated_files_count\": null,\n", " \"markers_count_plan\": null\n", " },\n", " \"snapshot\": {\n", " \"task_id\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"total_marked_time\": 160905.7,\n", " \"objects_count\": 550,\n", " \"marked_count\": 500,\n", " \"total_skipped_objects_count\": 0,\n", " \"total_marked_objects_count\": 1777,\n", " \"total_accepted_objects_count\": 1777,\n", " \"total_rejected_objects_count\": 0,\n", " \"current_overlap\": 3,\n", " \"total_objects_count\": 1500,\n", " \"avg_object_count_marker\": 126.93,\n", " \"status\": \"DONE\",\n", " \"one_object_avg_time\": 90.55,\n", " \"active_markers\": 0,\n", " \"total_markers\": 14,\n", " \"finished_marker\": 0,\n", " \"pretending_marker\": 0,\n", " \"cost\": 14216.0,\n", " \"avg_quality\": 0.79,\n", " \"avg_consistency\": 0.8,\n", " \"avg_hour_profit\": 565186.09\n", " },\n", " \"progress\": {\n", " \"marked_count\": 500,\n", " \"objects_count\": 550,\n", " \"progress_percent\": 90.91\n", " },\n", " \"workload\": {\n", " \"current_overlap\": 3,\n", " \"active_markers\": 0,\n", " \"total_markers\": 14,\n", " \"avg_object_count_marker\": 126.93,\n", " \"one_object_avg_time\": 90.55,\n", " \"total_marked_time\": 160905.7\n", " },\n", " \"status_breakdown\": {\n", " \"accepted_objects\": 1777,\n", " \"rejected_objects\": 0,\n", " \"marked_objects\": 1777,\n", " \"skipped_objects\": 0\n", " },\n", " \"quality\": {\n", " \"summary\": {\n", " \"average_quality\": 0.79,\n", " \"average_consistency\": 0.8\n", " }\n", " },\n", " \"time_range_stats\": null,\n", " \"alerts\": [\n", " \"low_quality\"\n", " ],\n", " \"summary\": \"Task \\\"🤖 NLAP TEXT | Разметка DLAP\\\" state=DONE, marked=500/550, progress=90.91%, overlap=3, active_markers=0, avg_quality=0.79, avg_consistency=0.8\"\n", "}\n" ] } ], "source": [ "if not TASK_ID:\n", " print('Set TASK_ID in the setup cell before running the live task demo.')\n", "else:\n", " task_args = {\n", " 'task_id': TASK_ID,\n", " 'include_advanced': True,\n", " }\n", " if ORGANIZATION_ID:\n", " task_args['organization_id'] = ORGANIZATION_ID\n", "\n", " task_stats = await call_tool('get_task_stats', task_args)\n", " show('get_task_stats', task_stats)\n", "\n", " summary_args = {\n", " 'task_id': TASK_ID,\n", " 'include_quality': INCLUDE_QUALITY,\n", " }\n", " if ORGANIZATION_ID:\n", " summary_args['organization_id'] = ORGANIZATION_ID\n", " if FROM_DATE and TO_DATE:\n", " summary_args['from_date'] = FROM_DATE\n", " summary_args['to_date'] = TO_DATE\n", "\n", " task_summary = await call_tool('get_annotation_summary', summary_args)\n", " show('get_annotation_summary(task)', task_summary)\n", "\n", " if INCLUDE_QUALITY:\n", " quality_args = {\n", " 'task_ids': [TASK_ID],\n", " 'limit': QUALITY_LIMIT,\n", " }\n", " if ORGANIZATION_ID:\n", " quality_args['organization_id'] = ORGANIZATION_ID\n", " task_quality = await call_tool('get_project_quality_stats', quality_args)\n", " show('get_project_quality_stats(task)', task_quality)" ] }, { "cell_type": "markdown", "id": "c540db18", "metadata": {}, "source": [ "## 4. LLM + MCP tool-use demo\n", "\n", "This is intentionally a minimal example.\n", "\n", "The idea is simple:\n", "1. Pass MCP tools into the LLM request.\n", "2. If the model asks to call a tool, execute that MCP tool.\n", "3. Send the tool result back to the LLM.\n", "4. Get the final natural-language answer.\n", "\n", "Before running the cell below, set `OPENAI_API_KEY` in the environment or `.local/.env`.\n", "Optionally set `OPENAI_BASE_URL` and `OPENAI_MODEL` for any OpenAI-compatible endpoint." ] }, { "cell_type": "code", "execution_count": null, "id": "799b6a49", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== llm_first_response ===\n", "{\n", " \"content\": null,\n", " \"role\": \"assistant\",\n", " \"tool_calls\": [\n", " {\n", " \"function\": {\n", " \"arguments\": \"{\\\"organization_id\\\":\\\"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\\\",\\\"task_id\\\":\\\"21fa9026-7482-4125-9393-382107727cda\\\",\\\"limit\\\":3,\\\"sort_by\\\":\\\"avg_quality\\\",\\\"sort_order\\\":\\\"desc\\\",\\\"include_without_quality\\\":true,\\\"min_marked\\\":0,\\\"min_quality_count\\\":0}\",\n", " \"name\": \"get_marker_performance_stats\"\n", " },\n", " \"id\": \"call_ciUSXUN6S5HujDS6PyxqZAnx\",\n", " \"type\": \"function\"\n", " }\n", " ]\n", "}\n", "\n", "=== llm_tool_call:get_marker_performance_stats ===\n", "{\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"task_id\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"limit\": 3,\n", " \"sort_by\": \"avg_quality\",\n", " \"sort_order\": \"desc\",\n", " \"include_without_quality\": true,\n", " \"min_marked\": 0,\n", " \"min_quality_count\": 0\n", "}\n", "\n", "=== mcp_result:get_marker_performance_stats ===\n", "{\n", " \"entity_type\": \"task\",\n", " \"task\": {\n", " \"uid\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"name\": \"🤖 NLAP TEXT | Разметка DLAP\",\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"created_date\": \"2026-04-15T18:45:32.130250+03:00\",\n", " \"update_date\": \"2026-04-22T15:51:42.940953+03:00\",\n", " \"start_date\": \"2026-04-16T13:28:06.160329+03:00\",\n", " \"finished_date\": \"2026-04-22T15:51:41.060853+03:00\",\n", " \"state\": \"DONE\",\n", " \"overlap\": 3,\n", " \"price\": 8.0,\n", " \"estimated_files_count\": null,\n", " \"markers_count_plan\": null\n", " },\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"time_range\": {\n", " \"from_date\": \"2026-04-16\",\n", " \"to_date\": \"2026-04-22\",\n", " \"inferred\": true\n", " },\n", " \"filters\": {\n", " \"marker_ids\": [],\n", " \"min_marked\": 0,\n", " \"min_quality_count\": 0,\n", " \"include_without_quality\": true\n", " },\n", " \"sorting\": {\n", " \"sort_by\": \"avg_quality\",\n", " \"sort_order\": \"desc\",\n", " \"limit\": 3\n", " },\n", " \"total_rows\": 44,\n", " \"total_markers\": 14,\n", " \"matched_markers\": 14,\n", " \"returned_markers\": 3,\n", " \"summary\": {\n", " \"markers_count\": 14,\n", " \"average_quality\": 0.8062,\n", " \"average_consistency\": 0.7819,\n", " \"total_marked\": 1777,\n", " \"total_accepted\": 1777,\n", " \"total_rejected\": 0,\n", " \"top_markers\": [\n", " {\n", " \"marker_id\": \"b4f50ac7-bda6-4db2-bef4-d7116d088ddf\",\n", " \"display_name\": \"Рожков Алексей\",\n", " \"avg_quality\": 1.0\n", " },\n", " {\n", " \"marker_id\": \"d7d95a51-cb49-4597-9e65-2fb5f03bc71f\",\n", " \"display_name\": \"А. З. Т.\",\n", " \"avg_quality\": 0.8572\n", " },\n", " {\n", " \"marker_id\": \"1560d4a2-4242-4725-9282-ee090a6c718d\",\n", " \"display_name\": \"В. Б. Н.\",\n", " \"avg_quality\": 0.8507\n", " }\n", " ],\n", " \"text\": \"Top markers by avg_quality: Рожков Алексей (avg_quality=1.0), А. З. Т. (avg_quality=0.8572), В. Б. Н. (avg_quality=0.8507)\"\n", " },\n", " \"items\": [\n", " {\n", " \"marker_id\": \"b4f50ac7-bda6-4db2-bef4-d7116d088ddf\",\n", " \"display_name\": \"Рожков Алексей\",\n", " \"fio\": \"Рожков Алексей\",\n", " \"email\": \"abbn81@gmail.com\",\n", " \"rows_count\": 2,\n", " \"marked\": 49,\n", " \"accepted\": 49,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 3,\n", " \"count_quality\": 3,\n", " \"overall_time\": 2.2369,\n", " \"sum_price\": 392.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-20T11:32:39\",\n", " \"end\": \"2026-04-21T16:56:51\",\n", " \"avg_quality\": 1.0,\n", " \"avg_time\": 164.344,\n", " \"avg_consistency\": 0.7743,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " },\n", " {\n", " \"marker_id\": \"d7d95a51-cb49-4597-9e65-2fb5f03bc71f\",\n", " \"display_name\": \"А. З. Т.\",\n", " \"fio\": \"А. З. Т.\",\n", " \"email\": \"tazakiryanova@mail.ru\",\n", " \"rows_count\": 4,\n", " \"marked\": 17,\n", " \"accepted\": 17,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 9,\n", " \"count_quality\": 7,\n", " \"overall_time\": 0.3799,\n", " \"sum_price\": 136.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-19T12:35:36\",\n", " \"end\": \"2026-04-22T13:40:26\",\n", " \"avg_quality\": 0.8572,\n", " \"avg_time\": 75.9745,\n", " \"avg_consistency\": 0.7894,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " },\n", " {\n", " \"marker_id\": \"1560d4a2-4242-4725-9282-ee090a6c718d\",\n", " \"display_name\": \"В. Б. Н.\",\n", " \"fio\": \"В. Б. Н.\",\n", " \"email\": \"agrafena75@yandex.ru\",\n", " \"rows_count\": 6,\n", " \"marked\": 112,\n", " \"accepted\": 112,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 5,\n", " \"count_quality\": 16,\n", " \"overall_time\": 2.1831,\n", " \"sum_price\": 896.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-16T19:26:37\",\n", " \"end\": \"2026-04-21T17:49:46\",\n", " \"avg_quality\": 0.8507,\n", " \"avg_time\": 70.1692,\n", " \"avg_consistency\": 0.8442,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " }\n", " ]\n", "}\n", "\n", "=== llm_final_answer ===\n", "{\n", " \"answer\": \"Топ 3 разметчика по качеству для TASK_ID `21fa9026-7482-4125-9393-382107727cda`:\\n\\n| # | Разметчик | Размечено заданий | Качество по контрольным заданиям | Качество по согласованности |\\n|---|---|---:|---:|---:|\\n| 1 | Рожков Алексей | 49 | 1.0000 | 0.7743 |\\n| 2 | А. З. Т. | 17 | 0.8572 | 0.7894 |\\n| 3 | В. Б. Н. | 112 | 0.8507 | 0.8442 |\\n\\nЕсли нужно, могу также вывести:\\n- email каждого разметчика,\\n- число контрольных заданий,\\n- средние показатели по всей задаче.\"\n", "}\n" ] } ], "source": [ "load_env(ENV_FILE)\n", "\n", "api_key = os.getenv('OPENAI_API_KEY')\n", "base_url = os.getenv('OPENAI_BASE_URL', 'https://api.openai.com/v1').rstrip('/')\n", "model = os.getenv('OPENAI_MODEL', 'gpt-5-mini')\n", "\n", "test_prompt = f\"\"\"Answer in Russian. Tell me the current status, progress and quality\n", " signals for task {TASK_ID}. organization_id: {ORGANIZATION_ID}\"\"\"\n", "\n", "test_prompt = f\"\"\"Выведи топ 3 разметчика по качеству для TASK_ID: {TASK_ID}. ORGANIZATION_ID: {ORGANIZATION_ID}.\n", "Для каждого из разметчиков выведи количество размеченных заданий и оценку качества по контрольным заданиям и качество по согласованности\"\"\"\n", "\n", "if not api_key:\n", " print('Set OPENAI_API_KEY in the environment or .local/.env before running this demo.')\n", "elif not TASK_ID:\n", " print('Set TASK_ID in the setup cell before running the LLM + MCP demo.')\n", "else:\n", " tools = await list_openai_tools()\n", " messages = [\n", " {\n", " 'role': 'system',\n", " 'content': 'You are a demo assistant for Crowd SDK. Use tools when you need real data.',\n", " },\n", " {\n", " 'role': 'user',\n", " 'content': test_prompt,\n", " },\n", " ]\n", "\n", " first_response = openai_chat_completion(messages, tools, model, api_key, base_url)\n", " first_message = first_response['choices'][0]['message']\n", " show('llm_first_response', first_message)\n", "\n", " tool_calls = first_message.get('tool_calls') or []\n", " if not tool_calls:\n", " show('llm_final_answer', {'answer': first_message.get('content', '')})\n", " else:\n", " messages.append(\n", " {\n", " 'role': 'assistant',\n", " 'content': first_message.get('content') or '',\n", " 'tool_calls': tool_calls,\n", " }\n", " )\n", "\n", " for tool_call in tool_calls:\n", " tool_name = tool_call['function']['name']\n", " tool_args = json.loads(tool_call['function'].get('arguments') or '{}')\n", " show(f'llm_tool_call:{tool_name}', tool_args)\n", "\n", " tool_result = await call_tool(tool_name, tool_args)\n", " show(f'mcp_result:{tool_name}', tool_result)\n", "\n", " messages.append(\n", " {\n", " 'role': 'tool',\n", " 'tool_call_id': tool_call['id'],\n", " 'content': json.dumps(tool_result, ensure_ascii=False),\n", " }\n", " )\n", "\n", " final_response = openai_chat_completion(messages, tools, model, api_key, base_url)\n", " final_message = final_response['choices'][0]['message']\n", " show('llm_final_answer', {'answer': final_message.get('content', '')})" ] }, { "cell_type": "code", "execution_count": null, "id": "b1d5d89f", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "=== llm_first_response ===\n", "{\n", " \"content\": null,\n", " \"role\": \"assistant\",\n", " \"tool_calls\": [\n", " {\n", " \"function\": {\n", " \"arguments\": \"{\\\"organization_id\\\":\\\"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\\\",\\\"task_id\\\":\\\"21fa9026-7482-4125-9393-382107727cda\\\",\\\"limit\\\":3,\\\"sort_by\\\":\\\"avg_quality\\\",\\\"sort_order\\\":\\\"desc\\\",\\\"include_without_quality\\\":true}\",\n", " \"name\": \"get_marker_performance_stats\"\n", " },\n", " \"id\": \"call_RkdhbjVmdQWuB9PEbpqEA52U\",\n", " \"type\": \"function\"\n", " }\n", " ]\n", "}\n", "\n", "=== llm_tool_call:get_marker_performance_stats ===\n", "{\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"task_id\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"limit\": 3,\n", " \"sort_by\": \"avg_quality\",\n", " \"sort_order\": \"desc\",\n", " \"include_without_quality\": true\n", "}\n", "\n", "=== mcp_result:get_marker_performance_stats ===\n", "{\n", " \"entity_type\": \"task\",\n", " \"task\": {\n", " \"uid\": \"21fa9026-7482-4125-9393-382107727cda\",\n", " \"name\": \"🤖 NLAP TEXT | Разметка DLAP\",\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"created_date\": \"2026-04-15T18:45:32.130250+03:00\",\n", " \"update_date\": \"2026-04-22T15:51:42.940953+03:00\",\n", " \"start_date\": \"2026-04-16T13:28:06.160329+03:00\",\n", " \"finished_date\": \"2026-04-22T15:51:41.060853+03:00\",\n", " \"state\": \"DONE\",\n", " \"overlap\": 3,\n", " \"price\": 8.0,\n", " \"estimated_files_count\": null,\n", " \"markers_count_plan\": null\n", " },\n", " \"project_id\": \"180f3050-7a75-4dfb-a883-735b12d8145e\",\n", " \"organization_id\": \"8f9375d6-4f3a-48d5-9a03-ac4e3977cb63\",\n", " \"time_range\": {\n", " \"from_date\": \"2026-04-16\",\n", " \"to_date\": \"2026-04-22\",\n", " \"inferred\": true\n", " },\n", " \"filters\": {\n", " \"marker_ids\": [],\n", " \"min_marked\": 0,\n", " \"min_quality_count\": 0,\n", " \"include_without_quality\": true\n", " },\n", " \"sorting\": {\n", " \"sort_by\": \"avg_quality\",\n", " \"sort_order\": \"desc\",\n", " \"limit\": 3\n", " },\n", " \"total_rows\": 44,\n", " \"total_markers\": 14,\n", " \"matched_markers\": 14,\n", " \"returned_markers\": 3,\n", " \"summary\": {\n", " \"markers_count\": 14,\n", " \"average_quality\": 0.8062,\n", " \"average_consistency\": 0.7819,\n", " \"total_marked\": 1777,\n", " \"total_accepted\": 1777,\n", " \"total_rejected\": 0,\n", " \"top_markers\": [\n", " {\n", " \"marker_id\": \"b4f50ac7-bda6-4db2-bef4-d7116d088ddf\",\n", " \"display_name\": \"Рожков Алексей\",\n", " \"avg_quality\": 1.0\n", " },\n", " {\n", " \"marker_id\": \"d7d95a51-cb49-4597-9e65-2fb5f03bc71f\",\n", " \"display_name\": \"А. З. Т.\",\n", " \"avg_quality\": 0.8572\n", " },\n", " {\n", " \"marker_id\": \"1560d4a2-4242-4725-9282-ee090a6c718d\",\n", " \"display_name\": \"Бурында Надежда\",\n", " \"avg_quality\": 0.8507\n", " }\n", " ],\n", " \"text\": \"Top markers by avg_quality: Рожков Алексей (avg_quality=1.0), А. З. Т. (avg_quality=0.8572), Бурында Надежда (avg_quality=0.8507)\"\n", " },\n", " \"items\": [\n", " {\n", " \"marker_id\": \"b4f50ac7-bda6-4db2-bef4-d7116d088ddf\",\n", " \"display_name\": \"Рожков Алексей\",\n", " \"fio\": \"Рожков Алексей\",\n", " \"email\": \"abbn81@gmail.com\",\n", " \"rows_count\": 2,\n", " \"marked\": 49,\n", " \"accepted\": 49,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 3,\n", " \"count_quality\": 3,\n", " \"overall_time\": 2.2369,\n", " \"sum_price\": 392.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-20T11:32:39\",\n", " \"end\": \"2026-04-21T16:56:51\",\n", " \"avg_quality\": 1.0,\n", " \"avg_time\": 164.344,\n", " \"avg_consistency\": 0.7743,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " },\n", " {\n", " \"marker_id\": \"d7d95a51-cb49-4597-9e65-2fb5f03bc71f\",\n", " \"display_name\": \"А. З. Т.\",\n", " \"fio\": \"А. З. Т.\",\n", " \"email\": \"tazakiryanova@mail.ru\",\n", " \"rows_count\": 4,\n", " \"marked\": 17,\n", " \"accepted\": 17,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 9,\n", " \"count_quality\": 7,\n", " \"overall_time\": 0.3799,\n", " \"sum_price\": 136.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-19T12:35:36\",\n", " \"end\": \"2026-04-22T13:40:26\",\n", " \"avg_quality\": 0.8572,\n", " \"avg_time\": 75.9745,\n", " \"avg_consistency\": 0.7894,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " },\n", " {\n", " \"marker_id\": \"1560d4a2-4242-4725-9282-ee090a6c718d\",\n", " \"display_name\": \"Бурында Надежда\",\n", " \"fio\": \"Бурында Надежда\",\n", " \"email\": \"agrafena75@yandex.ru\",\n", " \"rows_count\": 6,\n", " \"marked\": 112,\n", " \"accepted\": 112,\n", " \"rejected\": 0,\n", " \"submitted\": 0,\n", " \"skipped\": 0,\n", " \"expired\": 5,\n", " \"count_quality\": 16,\n", " \"overall_time\": 2.1831,\n", " \"sum_price\": 896.0,\n", " \"blocked_price\": 0.0,\n", " \"rejected_price\": 0.0,\n", " \"bonus_plus_penalty\": 0.0,\n", " \"begin\": \"2026-04-16T19:26:37\",\n", " \"end\": \"2026-04-21T17:49:46\",\n", " \"avg_quality\": 0.8507,\n", " \"avg_time\": 70.1692,\n", " \"avg_consistency\": 0.8442,\n", " \"acceptance_rate\": 100.0,\n", " \"rejection_rate\": 0.0\n", " }\n", " ]\n", "}\n", "\n", "=== llm_final_answer ===\n", "{\n", " \"answer\": \"Вот топ-3 разметчика по качеству для TASK_ID `21fa9026-7482-4125-9393-382107727cda`:\\n\\n| # | Разметчик | Размечено заданий | Качество по контрольным заданиям | Качество по согласованности |\\n|---|---|---:|---:|---:|\\n| 1 | Рожков Алексей | 49 | 1.0000 | 0.7743 |\\n| 2 | А. З. Т. | 17 | 0.8572 | 0.7894 |\\n| 3 | Бурында Надежда | 112 | 0.8507 | 0.8442 |\\n\\nЕсли хочешь, могу ещё вывести:\\n- их email,\\n- число принятых/отклонённых,\\n- среднее время на задание,\\n- или полный рейтинг всех 14 разметчиков по задаче.\"\n", "}\n" ] } ], "source": [ "load_env(ENV_FILE)\n", "\n", "api_key = os.getenv('OPENAI_API_KEY')\n", "base_url = os.getenv('OPENAI_BASE_URL', 'https://api.openai.com/v1').rstrip('/')\n", "model = os.getenv('OPENAI_MODEL', 'gpt-5-mini')\n", "\n", "test_prompt = f\"\"\"Answer in Russian. Tell me the current status, progress and quality\n", " signals for task {TASK_ID}. organization_id: {ORGANIZATION_ID}\"\"\"\n", "\n", "test_prompt = f\"\"\"Выведи топ 3 разметчика по качеству для TASK_ID: {TASK_ID}. ORGANIZATION_ID: {ORGANIZATION_ID}.\n", "Для каждого из разметчиков выведи количество размеченных заданий и оценку качества по контрольным заданиям и качество по согласованности\"\"\"\n", "\n", "if not api_key:\n", " print('Set OPENAI_API_KEY in the environment or .local/.env before running this demo.')\n", "elif not TASK_ID:\n", " print('Set TASK_ID in the setup cell before running the LLM + MCP demo.')\n", "else:\n", " tools = await list_openai_tools()\n", " messages = [\n", " {\n", " 'role': 'system',\n", " 'content': 'You are a demo assistant for Crowd SDK. Use tools when you need real data.',\n", " },\n", " {\n", " 'role': 'user',\n", " 'content': test_prompt,\n", " },\n", " ]\n", "\n", " first_response = openai_chat_completion(messages, tools, model, api_key, base_url)\n", " first_message = first_response['choices'][0]['message']\n", " show('llm_first_response', first_message)\n", "\n", " tool_calls = first_message.get('tool_calls') or []\n", " if not tool_calls:\n", " show('llm_final_answer', {'answer': first_message.get('content', '')})\n", " else:\n", " messages.append(\n", " {\n", " 'role': 'assistant',\n", " 'content': first_message.get('content') or '',\n", " 'tool_calls': tool_calls,\n", " }\n", " )\n", "\n", " for tool_call in tool_calls:\n", " tool_name = tool_call['function']['name']\n", " tool_args = json.loads(tool_call['function'].get('arguments') or '{}')\n", " show(f'llm_tool_call:{tool_name}', tool_args)\n", "\n", " tool_result = await call_tool(tool_name, tool_args)\n", " show(f'mcp_result:{tool_name}', tool_result)\n", "\n", " messages.append(\n", " {\n", " 'role': 'tool',\n", " 'tool_call_id': tool_call['id'],\n", " 'content': json.dumps(tool_result, ensure_ascii=False),\n", " }\n", " )\n", "\n", " final_response = openai_chat_completion(messages, tools, model, api_key, base_url)\n", " final_message = final_response['choices'][0]['message']\n", " show('llm_final_answer', {'answer': final_message.get('content', '')})" ] }, { "cell_type": "code", "execution_count": 16, "id": "3c848530", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Топ 3 разметчика по качеству для TASK_ID `21fa9026-7482-4125-9393-382107727cda`:\n", "\n", "| # | Разметчик | Размечено заданий | Качество по контрольным заданиям | Качество по согласованности |\n", "|---|---|---:|---:|---:|\n", "| 1 | Рожков Алексей | 49 | 1.0000 | 0.7743 |\n", "| 2 | А. З. Т. | 17 | 0.8572 | 0.7894 |\n", "| 3 | В. Б. Н. | 112 | 0.8507 | 0.8442 |\n", "\n", "Если нужно, могу также вывести:\n", "- email каждого разметчика,\n", "- число контрольных заданий,\n", "- средние показатели по всей задаче.\n" ] } ], "source": [ "print(final_message.get('content', ''))" ] }, { "cell_type": "code", "execution_count": null, "id": "0f9d3af7", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": ".venv-mcp-check (3.12.13)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.13" } }, "nbformat": 4, "nbformat_minor": 5 }