{ "Environment": { "$defs": { "AppAction": { "additionalProperties": false, "description": "One move an App offers the Agent.\n\n``name`` is the App's own name for the action (a Python method name or an\nMCP tool name), which is how the App is called. The Agent sees it under\nits exposed name: the same name, with any character a model provider\nrejects replaced, and prefixed ``__`` only when another App offers\nthe same name.", "properties": { "name": { "maxLength": 128, "minLength": 1, "title": "Name", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "parameters": { "additionalProperties": true, "title": "Parameters", "type": "object" }, "mutates_state": { "default": true, "title": "Mutates State", "type": "boolean" }, "timeout_seconds": { "default": 30, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" }, "binding": { "additionalProperties": true, "title": "Binding", "type": "object" } }, "required": [ "name" ], "title": "AppAction", "type": "object" }, "AppDefinition": { "additionalProperties": false, "description": "One App inside an Environment.\n\n``python`` names the App class as ``file.py:Class`` inside the Environment's\nsource package. ``config`` holds the keyword arguments the App was built\nwith, so the App host can rebuild the same App inside a Runtime.", "properties": { "name": { "title": "Name", "type": "string" }, "kind": { "default": "python", "enum": [ "python", "mcp", "openapi", "replay" ], "title": "Kind", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "python": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Python" }, "config": { "additionalProperties": true, "title": "Config", "type": "object" }, "actions": { "default": [], "items": { "$ref": "#/$defs/AppAction" }, "title": "Actions", "type": "array" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "credentials": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Credentials", "type": "array" }, "state_schema": { "additionalProperties": true, "title": "State Schema", "type": "object" }, "has_reset": { "default": false, "title": "Has Reset", "type": "boolean" }, "has_snapshot": { "default": false, "title": "Has Snapshot", "type": "boolean" }, "control": { "default": "owned", "enum": [ "owned", "sandboxed", "live", "replayed" ], "title": "Control", "type": "string" } }, "required": [ "name" ], "title": "AppDefinition", "type": "object" }, "DeclarativeImage": { "additionalProperties": false, "description": "Portable subset of Daytona's declarative image builder.", "properties": { "base": { "minLength": 1, "title": "Base", "type": "string" }, "pip_packages": { "default": [], "items": { "type": "string" }, "title": "Pip Packages", "type": "array" }, "environment": { "additionalProperties": { "type": "string" }, "title": "Environment", "type": "object" }, "workdir": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Workdir" } }, "required": [ "base" ], "title": "DeclarativeImage", "type": "object" }, "EnvironmentResource": { "additionalProperties": false, "description": "Data or application supplied by an Environment.\n\nThe short form ``Resource(\"policies/refunds.md\")`` stages one file from\nthe Environment's source package. The mapping form keeps its explicit\n``delivery``. ``Resource.from_file(path)`` reads a file of any type: text\ntravels inline, and anything else, such as a PDF or a drawing, as a\n``blob`` named by its digest.", "properties": { "kind": { "enum": [ "data", "application", "file" ], "title": "Kind", "type": "string" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Path" }, "uri": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Uri" }, "digest": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Digest" }, "content_type": { "default": "", "title": "Content Type", "type": "string" }, "config": { "additionalProperties": true, "title": "Config", "type": "object" }, "description": { "default": "", "title": "Description", "type": "string" }, "delivery": { "default": "descriptor", "enum": [ "descriptor", "source", "inline", "resolver", "blob" ], "title": "Delivery", "type": "string" }, "content": { "anyOf": [ { "maxLength": 1048576, "type": "string" }, { "type": "null" } ], "default": null, "title": "Content" }, "resolver": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Resolver" } }, "required": [ "kind", "name" ], "title": "EnvironmentResource", "type": "object" }, "EnvironmentRuntime": { "additionalProperties": false, "description": "Where an Environment runs: Docker, a trusted local process, or a remote sandbox.", "properties": { "provider": { "default": "docker", "minLength": 1, "title": "Provider", "type": "string" }, "ref": { "anyOf": [ { "pattern": "^[a-z0-9][a-z0-9-]{0,79}$", "type": "string" }, { "type": "null" } ], "default": null, "description": "Slug of the project Runtime this configuration was copied from. The machine fields hold that Runtime's settings as of the copy; credentials come from the project Runtime when a Job starts.", "title": "Ref" }, "placement": { "additionalProperties": { "type": "string" }, "title": "Placement", "type": "object" }, "variables": { "default": [], "items": { "$ref": "#/$defs/RuntimeVariable" }, "title": "Variables", "type": "array" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Image" }, "snapshot": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Snapshot" }, "declarative_image": { "anyOf": [ { "$ref": "#/$defs/DeclarativeImage" }, { "type": "null" } ], "default": null }, "build_context": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Build Context" }, "dockerfile": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Dockerfile" }, "network": { "$ref": "#/$defs/NetworkMode", "default": "public" }, "allowed_hosts": { "default": [], "description": "Hosts the Runtime may reach when network is 'allowlist'.", "items": { "type": "string" }, "title": "Allowed Hosts", "type": "array" }, "compute": { "$ref": "#/$defs/ResourceRequirements", "description": "CPU, memory, storage, and process limits for the Runtime." }, "packages": { "$ref": "#/$defs/RuntimePackages", "description": "Packages installed in the Runtime before an episode. In environment.yaml, list Python ones under python_packages. On the local Runtime they must already be installed." }, "read_only_root": { "default": false, "title": "Read Only Root", "type": "boolean" }, "targets": { "items": { "$ref": "#/$defs/ExecutionTarget" }, "title": "Targets", "type": "array", "uniqueItems": true }, "persistent": { "default": false, "title": "Persistent", "type": "boolean" }, "compose": { "default": false, "title": "Compose", "type": "boolean" }, "extra_capabilities": { "default": [], "items": { "$ref": "#/$defs/SandboxCapability" }, "title": "Extra Capabilities", "type": "array", "uniqueItems": true }, "timeout_seconds": { "default": 300, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" }, "build_timeout_seconds": { "default": 600, "exclusiveMinimum": 0, "title": "Build Timeout Seconds", "type": "number" }, "allow_unsafe_local": { "default": false, "title": "Allow Unsafe Local", "type": "boolean" } }, "title": "EnvironmentRuntime", "type": "object" }, "ExecutionLimits": { "additionalProperties": false, "description": "Environment execution limits.", "properties": { "max_turns": { "default": 8, "maximum": 1000, "minimum": 1, "title": "Max Turns", "type": "integer" }, "max_seconds": { "default": 120, "exclusiveMinimum": 0, "title": "Max Seconds", "type": "number" }, "max_cost_usd": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Max Cost Usd" } }, "title": "ExecutionLimits", "type": "object" }, "ExecutionTarget": { "description": "Execution isolation class.", "enum": [ "local", "docker", "remote" ], "title": "ExecutionTarget", "type": "string" }, "Guardrail": { "additionalProperties": false, "description": "A plain-language Environment rule.", "properties": { "name": { "default": "", "title": "Name", "type": "string" }, "rule": { "minLength": 1, "title": "Rule", "type": "string" } }, "required": [ "rule" ], "title": "Guardrail", "type": "object" }, "HarnessCapability": { "description": "A capability requested by an agent harness.", "enum": [ "shell", "file_read", "file_edit", "code_execution", "web_search", "browser", "network_fetch", "mcp", "subagents", "persistence" ], "title": "HarnessCapability", "type": "string" }, "HarnessPolicy": { "additionalProperties": false, "description": "Environment ceiling for a Trial's optional Agent harness.", "properties": { "mode": { "default": "allow_all", "enum": [ "allow_all", "allowlist" ], "title": "Mode", "type": "string" }, "allowed_harnesses": { "default": [], "items": { "type": "string" }, "title": "Allowed Harnesses", "type": "array" }, "allowed_capabilities": { "anyOf": [ { "items": { "$ref": "#/$defs/HarnessCapability" }, "type": "array", "uniqueItems": true }, { "type": "null" } ], "default": null, "title": "Allowed Capabilities" }, "denied_capabilities": { "default": [], "items": { "$ref": "#/$defs/HarnessCapability" }, "title": "Denied Capabilities", "type": "array", "uniqueItems": true } }, "title": "HarnessPolicy", "type": "object" }, "NetworkMode": { "description": "Requested sandbox network policy.\n\nUse public, no-network, or allowlist to configure outbound access.", "enum": [ "public", "no-network", "allowlist", "public", "no-network", "allowlist" ], "title": "NetworkMode", "type": "string" }, "Party": { "additionalProperties": false, "description": "How many Agents may share one episode of the Environment, such as 2 to 8 at poker.\n\nAn Environment with a party is played by several Agents at once, each in a\nseat of its own: ``seat_1`` to ``seat_N`` for a party of N. A Job names the\nAgents in the party, and its size must fall between ``min`` and ``max``.", "properties": { "min": { "default": 2, "maximum": 100, "minimum": 2, "title": "Min", "type": "integer" }, "max": { "default": 2, "maximum": 100, "minimum": 2, "title": "Max", "type": "integer" }, "isolation": { "default": "isolated", "description": "``isolated``: each seat's Harness runs as its own OS user, and the Apps are the only thing the seats share; needs Docker or Daytona. ``shared``: every seat runs as one user and can read and write the others' files and the Environment's, for a team that works on the same files, or to try a party on the local Runtime. A shared seat could also act for another seat.", "enum": [ "isolated", "shared" ], "title": "Isolation", "type": "string" } }, "title": "Party", "type": "object" }, "ResourceRequirements": { "additionalProperties": false, "description": "Optional compute limits.", "properties": { "cpu": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Cpu" }, "memory_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Memory Mb" }, "pids": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Pids" }, "disk_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Disk Mb" }, "storage_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Storage Mb" } }, "title": "ResourceRequirements", "type": "object" }, "RuntimePackages": { "additionalProperties": false, "description": "Packages installed in the Runtime before an episode, by package manager.\n\nEach list holds what you would pass to that manager, one package per entry.", "properties": { "python": { "default": [], "items": { "type": "string" }, "title": "Python", "type": "array" } }, "title": "RuntimePackages", "type": "object" }, "RuntimeVariable": { "additionalProperties": false, "description": "A launch-time variable declaration. Values are supplied to Job, never saved here.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "secret": { "default": false, "title": "Secret", "type": "boolean" }, "format": { "default": "text", "enum": [ "text", "url", "integer", "json" ], "title": "Format", "type": "string" } }, "required": [ "name" ], "title": "RuntimeVariable", "type": "object" }, "SandboxCapability": { "description": "Individual controls a provider can enforce.", "enum": [ "image", "build", "resources", "network_none", "network_allowlist", "upload", "download", "timeout", "cancel", "persistence", "compose", "read_only_root", "working_directory", "environment", "log_capture" ], "title": "SandboxCapability", "type": "string" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" } }, "additionalProperties": false, "description": "Schema-v2 Environment: Apps composed with one Runtime and one set of limits.\n\nTasks, Verifiers, and mode are intentionally absent. Every action belongs\nto exactly one App; ``python`` optionally names an Environment subclass\nwhose ``on_step``, ``terminated``, or ``truncated`` span several Apps.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "overview": { "default": "", "title": "Overview", "type": "string" }, "readme": { "default": "", "title": "Readme", "type": "string" }, "apps": { "default": [], "items": { "$ref": "#/$defs/AppDefinition" }, "title": "Apps", "type": "array" }, "python": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Python" }, "guardrails": { "default": [], "items": { "$ref": "#/$defs/Guardrail" }, "title": "Guardrails", "type": "array" }, "resources": { "default": [], "items": { "$ref": "#/$defs/EnvironmentResource" }, "title": "Resources", "type": "array" }, "runtime": { "$ref": "#/$defs/EnvironmentRuntime" }, "harness_policy": { "$ref": "#/$defs/HarnessPolicy" }, "limits": { "$ref": "#/$defs/ExecutionLimits" }, "party": { "anyOf": [ { "$ref": "#/$defs/Party" }, { "type": "null" } ], "default": null, "description": "How many Agents share each episode, for a game such as poker. Leave it unset for an Environment one Agent works in alone." }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" } }, "required": [ "name" ], "title": "Environment", "type": "object" }, "Harness": { "additionalProperties": false, "description": "Base class for an Agent interaction loop.\n\nSubclasses implement :meth:`run`. Plural discovers and hashes the Python\nclass directory, invokes ``run`` in the Environment Runtime, and emits the\nstandard result, trajectory, and log artifacts.", "properties": { "config": { "additionalProperties": true, "title": "Config", "type": "object" } }, "title": "Harness", "type": "object" }, "Task": { "$defs": { "AgentVerifier": { "additionalProperties": false, "description": "Model-judge verifier with its own runtime and connectivity policy.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "agent", "default": "agent", "title": "Kind", "type": "string" }, "model": { "minLength": 1, "title": "Model", "type": "string" }, "instructions": { "minLength": 1, "title": "Instructions", "type": "string" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Provider" }, "fallback_models": { "default": [], "items": { "type": "string" }, "title": "Fallback Models", "type": "array" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" } }, "required": [ "name", "criteria", "model", "instructions" ], "title": "AgentVerifier", "type": "object" }, "DeterministicVerifier": { "additionalProperties": false, "anyOf": [ { "required": [ "check" ] }, { "required": [ "command" ] }, { "required": [ "reward" ] }, { "required": [ "hindsight" ] } ], "description": "Code that verifies an episode: it scores the end, rewards actions, or both.\n\n``check`` scores the finished episode; that score is what Tasks and\nBenchmarks rank on. It is a function of the :class:`Episode`, a\n``{\"python\": \"verify.py:check\"}`` reference, or a command argv.\n\n``reward`` rewards each action while the episode runs. It is a function of\none :class:`Step`, or a ``{\"python\": \"verify.py:reward\"}`` reference, and\nruns in the Runtime beside the Apps, so it can read every App. What it\nreturns goes to the Trial's reward ledger. Rewards reach the score only\nwhen ``check`` is :func:`total_reward`.\n\n``hindsight`` credits the finished trajectory as a whole: a function of a\n:class:`~plural.rewards.HindsightTrial` that returns\n:class:`~plural.rewards.Credit` values, or a ``{\"python\":\n\"verify.py:efficiency\"}`` reference. It runs after the score, and can also\nrun later on stored Trials with ``plural verifier apply``.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "default": [], "items": { "$ref": "#/$defs/RubricCriterion" }, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "deterministic", "default": "deterministic", "title": "Kind", "type": "string" }, "check": { "default": null, "title": "Check" }, "reward": { "default": null, "title": "Reward" }, "hindsight": { "default": null, "title": "Hindsight" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" }, "result_path": { "default": "verifier-result.json", "title": "Result Path", "type": "string" }, "evidence_required": { "default": true, "title": "Evidence Required", "type": "boolean" } }, "required": [ "name" ], "title": "DeterministicVerifier", "type": "object" }, "EnvironmentResource": { "additionalProperties": false, "description": "Data or application supplied by an Environment.\n\nThe short form ``Resource(\"policies/refunds.md\")`` stages one file from\nthe Environment's source package. The mapping form keeps its explicit\n``delivery``. ``Resource.from_file(path)`` reads a file of any type: text\ntravels inline, and anything else, such as a PDF or a drawing, as a\n``blob`` named by its digest.", "properties": { "kind": { "enum": [ "data", "application", "file" ], "title": "Kind", "type": "string" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Path" }, "uri": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Uri" }, "digest": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Digest" }, "content_type": { "default": "", "title": "Content Type", "type": "string" }, "config": { "additionalProperties": true, "title": "Config", "type": "object" }, "description": { "default": "", "title": "Description", "type": "string" }, "delivery": { "default": "descriptor", "enum": [ "descriptor", "source", "inline", "resolver", "blob" ], "title": "Delivery", "type": "string" }, "content": { "anyOf": [ { "maxLength": 1048576, "type": "string" }, { "type": "null" } ], "default": null, "title": "Content" }, "resolver": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Resolver" } }, "required": [ "kind", "name" ], "title": "EnvironmentResource", "type": "object" }, "HumanVerifier": { "additionalProperties": false, "description": "Human review verifier.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "human", "default": "human", "title": "Kind", "type": "string" }, "instructions": { "default": "", "title": "Instructions", "type": "string" } }, "required": [ "name", "criteria" ], "title": "HumanVerifier", "type": "object" }, "NetworkMode": { "description": "Requested sandbox network policy.\n\nUse public, no-network, or allowlist to configure outbound access.", "enum": [ "public", "no-network", "allowlist", "public", "no-network", "allowlist" ], "title": "NetworkMode", "type": "string" }, "ResourceRequirements": { "additionalProperties": false, "description": "Optional compute limits.", "properties": { "cpu": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Cpu" }, "memory_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Memory Mb" }, "pids": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Pids" }, "disk_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Disk Mb" }, "storage_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Storage Mb" } }, "title": "ResourceRequirements", "type": "object" }, "RubricCriterion": { "additionalProperties": false, "description": "One Agent or Human rubric criterion.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "minLength": 1, "title": "Description", "type": "string" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "min_score": { "default": 0, "title": "Min Score", "type": "number" }, "max_score": { "default": 1, "title": "Max Score", "type": "number" } }, "required": [ "name", "description" ], "title": "RubricCriterion", "type": "object" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" }, "VerifierRuntime": { "additionalProperties": false, "description": "Machine for a verifier that needs its own sandbox.\n\nThe default runs in the Task Environment. Set a provider, image, network,\nresource limit, or timeout to use a different machine.", "properties": { "provider": { "default": "docker", "minLength": 1, "title": "Provider", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Image" }, "network": { "$ref": "#/$defs/NetworkMode", "default": "public" }, "allowed_hosts": { "default": [], "items": { "type": "string" }, "title": "Allowed Hosts", "type": "array" }, "compute": { "$ref": "#/$defs/ResourceRequirements" }, "timeout_seconds": { "default": 60, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" } }, "title": "VerifierRuntime", "type": "object" } }, "additionalProperties": false, "description": "One versioned unit of work in a Python-authored Environment.\n\nIn an Environment with a party, several Agents share each episode, one\nper seat. Each gets ``instructions``, and then its seat's entry in\n``seat_instructions``, keyed ``seat_1``, ``seat_2``, and so on.\n\n``expected`` is the answer key: what a correct episode produces. Verifiers\nread it as ``episode.expected``; it never reaches the Agent, unlike ``info``\nand ``metadata``.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "instructions": { "minLength": 1, "title": "Instructions", "type": "string" }, "goals": { "default": [], "items": { "type": "string" }, "title": "Goals", "type": "array" }, "info": { "default": null, "title": "Info" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "environment": { "title": "Environment" }, "verifiers": { "items": { "discriminator": { "mapping": { "agent": "#/$defs/AgentVerifier", "deterministic": "#/$defs/DeterministicVerifier", "human": "#/$defs/HumanVerifier" }, "propertyName": "kind" }, "oneOf": [ { "$ref": "#/$defs/DeterministicVerifier" }, { "$ref": "#/$defs/AgentVerifier" }, { "$ref": "#/$defs/HumanVerifier" } ] }, "minItems": 1, "title": "Verifiers", "type": "array" }, "resources": { "default": [], "items": { "$ref": "#/$defs/EnvironmentResource" }, "title": "Resources", "type": "array" }, "initial_state": { "additionalProperties": true, "title": "Initial State", "type": "object" }, "seat_instructions": { "additionalProperties": { "type": "string" }, "title": "Seat Instructions", "type": "object" }, "expected": { "default": null, "title": "Expected" } }, "required": [ "name", "instructions", "environment", "verifiers" ], "title": "Task", "type": "object" }, "DeterministicVerifier": { "$defs": { "NetworkMode": { "description": "Requested sandbox network policy.\n\nUse public, no-network, or allowlist to configure outbound access.", "enum": [ "public", "no-network", "allowlist", "public", "no-network", "allowlist" ], "title": "NetworkMode", "type": "string" }, "ResourceRequirements": { "additionalProperties": false, "description": "Optional compute limits.", "properties": { "cpu": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Cpu" }, "memory_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Memory Mb" }, "pids": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Pids" }, "disk_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Disk Mb" }, "storage_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Storage Mb" } }, "title": "ResourceRequirements", "type": "object" }, "RubricCriterion": { "additionalProperties": false, "description": "One Agent or Human rubric criterion.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "minLength": 1, "title": "Description", "type": "string" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "min_score": { "default": 0, "title": "Min Score", "type": "number" }, "max_score": { "default": 1, "title": "Max Score", "type": "number" } }, "required": [ "name", "description" ], "title": "RubricCriterion", "type": "object" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" }, "VerifierRuntime": { "additionalProperties": false, "description": "Machine for a verifier that needs its own sandbox.\n\nThe default runs in the Task Environment. Set a provider, image, network,\nresource limit, or timeout to use a different machine.", "properties": { "provider": { "default": "docker", "minLength": 1, "title": "Provider", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Image" }, "network": { "$ref": "#/$defs/NetworkMode", "default": "public" }, "allowed_hosts": { "default": [], "items": { "type": "string" }, "title": "Allowed Hosts", "type": "array" }, "compute": { "$ref": "#/$defs/ResourceRequirements" }, "timeout_seconds": { "default": 60, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" } }, "title": "VerifierRuntime", "type": "object" } }, "additionalProperties": false, "anyOf": [ { "required": [ "check" ] }, { "required": [ "command" ] }, { "required": [ "reward" ] }, { "required": [ "hindsight" ] } ], "description": "Code that verifies an episode: it scores the end, rewards actions, or both.\n\n``check`` scores the finished episode; that score is what Tasks and\nBenchmarks rank on. It is a function of the :class:`Episode`, a\n``{\"python\": \"verify.py:check\"}`` reference, or a command argv.\n\n``reward`` rewards each action while the episode runs. It is a function of\none :class:`Step`, or a ``{\"python\": \"verify.py:reward\"}`` reference, and\nruns in the Runtime beside the Apps, so it can read every App. What it\nreturns goes to the Trial's reward ledger. Rewards reach the score only\nwhen ``check`` is :func:`total_reward`.\n\n``hindsight`` credits the finished trajectory as a whole: a function of a\n:class:`~plural.rewards.HindsightTrial` that returns\n:class:`~plural.rewards.Credit` values, or a ``{\"python\":\n\"verify.py:efficiency\"}`` reference. It runs after the score, and can also\nrun later on stored Trials with ``plural verifier apply``.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "default": [], "items": { "$ref": "#/$defs/RubricCriterion" }, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "deterministic", "default": "deterministic", "title": "Kind", "type": "string" }, "check": { "default": null, "title": "Check" }, "reward": { "default": null, "title": "Reward" }, "hindsight": { "default": null, "title": "Hindsight" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" }, "result_path": { "default": "verifier-result.json", "title": "Result Path", "type": "string" }, "evidence_required": { "default": true, "title": "Evidence Required", "type": "boolean" } }, "required": [ "name" ], "title": "DeterministicVerifier", "type": "object" }, "AgentVerifier": { "$defs": { "NetworkMode": { "description": "Requested sandbox network policy.\n\nUse public, no-network, or allowlist to configure outbound access.", "enum": [ "public", "no-network", "allowlist", "public", "no-network", "allowlist" ], "title": "NetworkMode", "type": "string" }, "ResourceRequirements": { "additionalProperties": false, "description": "Optional compute limits.", "properties": { "cpu": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Cpu" }, "memory_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Memory Mb" }, "pids": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Pids" }, "disk_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Disk Mb" }, "storage_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Storage Mb" } }, "title": "ResourceRequirements", "type": "object" }, "RubricCriterion": { "additionalProperties": false, "description": "One Agent or Human rubric criterion.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "minLength": 1, "title": "Description", "type": "string" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "min_score": { "default": 0, "title": "Min Score", "type": "number" }, "max_score": { "default": 1, "title": "Max Score", "type": "number" } }, "required": [ "name", "description" ], "title": "RubricCriterion", "type": "object" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" }, "VerifierRuntime": { "additionalProperties": false, "description": "Machine for a verifier that needs its own sandbox.\n\nThe default runs in the Task Environment. Set a provider, image, network,\nresource limit, or timeout to use a different machine.", "properties": { "provider": { "default": "docker", "minLength": 1, "title": "Provider", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Image" }, "network": { "$ref": "#/$defs/NetworkMode", "default": "public" }, "allowed_hosts": { "default": [], "items": { "type": "string" }, "title": "Allowed Hosts", "type": "array" }, "compute": { "$ref": "#/$defs/ResourceRequirements" }, "timeout_seconds": { "default": 60, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" } }, "title": "VerifierRuntime", "type": "object" } }, "additionalProperties": false, "description": "Model-judge verifier with its own runtime and connectivity policy.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "agent", "default": "agent", "title": "Kind", "type": "string" }, "model": { "minLength": 1, "title": "Model", "type": "string" }, "instructions": { "minLength": 1, "title": "Instructions", "type": "string" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Provider" }, "fallback_models": { "default": [], "items": { "type": "string" }, "title": "Fallback Models", "type": "array" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" } }, "required": [ "name", "criteria", "model", "instructions" ], "title": "AgentVerifier", "type": "object" }, "HumanVerifier": { "$defs": { "RubricCriterion": { "additionalProperties": false, "description": "One Agent or Human rubric criterion.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "minLength": 1, "title": "Description", "type": "string" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "min_score": { "default": 0, "title": "Min Score", "type": "number" }, "max_score": { "default": 1, "title": "Max Score", "type": "number" } }, "required": [ "name", "description" ], "title": "RubricCriterion", "type": "object" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" } }, "additionalProperties": false, "description": "Human review verifier.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "human", "default": "human", "title": "Kind", "type": "string" }, "instructions": { "default": "", "title": "Instructions", "type": "string" } }, "required": [ "name", "criteria" ], "title": "HumanVerifier", "type": "object" }, "Agent": { "$defs": { "AgentCredential": { "additionalProperties": false, "description": "A credential this Agent supplies to the Apps it works with.\n\nThe value, such as the Salesforce token of the user this Agent acts as, is\nnever part of the Agent: each person who runs it saves their own with\n``plural credential set`` or exports it, and it reaches only the App host.\n``scopes`` records what the token may do, as you describe it; the system\nthat issued the token enforces them, and every Trial records them for audit.", "properties": { "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" }, "description": { "default": "", "title": "Description", "type": "string" } }, "title": "AgentCredential", "type": "object" }, "ExternalAgent": { "additionalProperties": false, "description": "An agent that already runs as a service of its own.\n\nA Job calls ``url`` with a signed request for each episode. The agent acts\nonly through that episode's actions and files, and its code is not part of\nany content hash: ``version`` is the label you give each release.", "properties": { "url": { "title": "Url", "type": "string" }, "signing_key": { "title": "Signing Key", "type": "string" }, "version": { "default": "", "title": "Version", "type": "string" }, "ca_bundle": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Ca Bundle" } }, "required": [ "url", "signing_key" ], "title": "ExternalAgent", "type": "object" }, "FileDeclaration": { "additionalProperties": false, "description": "A file emitted or consumed by a package.", "properties": { "path": { "minLength": 1, "title": "Path", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "media_type": { "default": "application/octet-stream", "title": "Media Type", "type": "string" } }, "required": [ "path" ], "title": "FileDeclaration", "type": "object" }, "Harness": { "additionalProperties": false, "description": "Base class for an Agent interaction loop.\n\nSubclasses implement :meth:`run`. Plural discovers and hashes the Python\nclass directory, invokes ``run`` in the Environment Runtime, and emits the\nstandard result, trajectory, and log artifacts.", "properties": { "config": { "additionalProperties": true, "title": "Config", "type": "object" } }, "title": "Harness", "type": "object" }, "HarnessCapability": { "description": "A capability requested by an agent harness.", "enum": [ "shell", "file_read", "file_edit", "code_execution", "web_search", "browser", "network_fetch", "mcp", "subagents", "persistence" ], "title": "HarnessCapability", "type": "string" }, "HarnessDefinition": { "additionalProperties": false, "description": "A declared executable Agent interaction strategy.\n\nPoint ``source`` at a local directory or a remote archive and name the\n``command`` that runs the loop. Subclass :class:`Harness` instead to write\none in Python.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "command": { "items": { "type": "string" }, "minItems": 1, "title": "Command", "type": "array" }, "source": { "default": ".", "title": "Source", "type": "string" }, "digest": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Digest" }, "requirements": { "default": [], "items": { "type": "string" }, "title": "Requirements", "type": "array" }, "capabilities": { "default": [], "items": { "$ref": "#/$defs/HarnessCapability" }, "title": "Capabilities", "type": "array", "uniqueItems": true }, "models": { "default": [ "*" ], "items": { "type": "string" }, "title": "Models", "type": "array" }, "auth": { "default": [ "environment" ], "items": { "enum": [ "environment", "api_key", "oauth", "none" ], "type": "string" }, "title": "Auth", "type": "array" }, "secrets": { "default": [], "items": { "type": "string" }, "title": "Secrets", "type": "array" }, "env_vars": { "default": [], "description": "Names of environment variables passed through to the Harness process.", "items": { "type": "string" }, "title": "Env Vars", "type": "array" }, "healthcheck": { "anyOf": [ { "items": { "type": "string" }, "type": "array" }, { "type": "null" } ], "default": null, "title": "Healthcheck" }, "outputs": { "default": [], "items": { "$ref": "#/$defs/FileDeclaration" }, "title": "Outputs", "type": "array" }, "artifacts": { "default": [], "items": { "$ref": "#/$defs/FileDeclaration" }, "title": "Artifacts", "type": "array" }, "trajectory": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Trajectory" }, "tito": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Tito" } }, "required": [ "name", "command" ], "title": "HarnessDefinition", "type": "object" } }, "additionalProperties": false, "description": "A catalog-backed model and its optional execution harness.\n\nTo evaluate your own agent or model server instead, give ``endpoint``: the\nbase URL of an OpenAI-compatible API (one that serves\n``/chat/completions``). ``model`` is then sent to it as is and need not be\nin the catalog, and ``endpoint_key`` names the secret holding its API key.\nPlural's native Harness calls it with the Environment's actions as tools,\nand a custom Harness reaches it through ``agent.complete``.\n\nAn agent that already runs elsewhere is an ``external`` Agent instead: the\nJob calls it over signed HTTPS, and ``model`` is the label it reports.", "properties": { "model": { "minLength": 1, "title": "Model", "type": "string" }, "name": { "default": "", "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Provider" }, "instructions": { "default": "", "title": "Instructions", "type": "string" }, "fallback_models": { "default": [], "items": { "type": "string" }, "title": "Fallback Models", "type": "array" }, "temperature": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "title": "Temperature" }, "max_tokens": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Max Tokens" }, "harness": { "anyOf": [ { "$ref": "#/$defs/HarnessDefinition" }, { "$ref": "#/$defs/Harness" }, { "type": "string" }, { "type": "null" } ], "default": null, "title": "Harness" }, "harness_config": { "additionalProperties": true, "description": "Options for a built-in Harness, such as reasoning_effort.", "title": "Harness Config", "type": "object" }, "auth": { "default": "environment", "description": "How the Harness authenticates to the model; none when it calls no model.", "enum": [ "environment", "api_key", "oauth", "none" ], "title": "Auth", "type": "string" }, "secrets": { "default": [], "description": "Names of the Harness's declared secrets this Agent grants it.", "items": { "type": "string" }, "title": "Secrets", "type": "array" }, "endpoint": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Endpoint" }, "endpoint_key": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Endpoint Key" }, "external": { "anyOf": [ { "$ref": "#/$defs/ExternalAgent" }, { "type": "null" } ], "default": null }, "credentials": { "additionalProperties": { "$ref": "#/$defs/AgentCredential" }, "title": "Credentials", "type": "object" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" } }, "required": [ "model" ], "title": "Agent", "type": "object" }, "Benchmark": { "$defs": { "AgentVerifier": { "additionalProperties": false, "description": "Model-judge verifier with its own runtime and connectivity policy.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "agent", "default": "agent", "title": "Kind", "type": "string" }, "model": { "minLength": 1, "title": "Model", "type": "string" }, "instructions": { "minLength": 1, "title": "Instructions", "type": "string" }, "provider": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Provider" }, "fallback_models": { "default": [], "items": { "type": "string" }, "title": "Fallback Models", "type": "array" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" } }, "required": [ "name", "criteria", "model", "instructions" ], "title": "AgentVerifier", "type": "object" }, "BenchmarkCategory": { "additionalProperties": false, "description": "A named group of Tasks, scored together in results.\n\n``id`` is stable across releases. A category is author-defined unless\n``taxonomy`` names a shared classification, such as ``\"plural:coding\"``.\nScores in author-defined categories are only comparable within this\nBenchmark; two Benchmarks that both say \"reasoning\" may measure different\nthings.\n\nAttributes:\n id: Stable identifier within the Benchmark, such as ``\"hard-words\"``.\n name: Label shown to readers.\n description: What the Tasks in this category have in common.\n tasks: Names of the Tasks in this category. A Task may be in several.\n taxonomy: Optional shared classification identifier.", "properties": { "id": { "title": "Id", "type": "string" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "tasks": { "items": { "type": "string" }, "minItems": 1, "title": "Tasks", "type": "array" }, "taxonomy": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Taxonomy" } }, "required": [ "id", "name", "tasks" ], "title": "BenchmarkCategory", "type": "object" }, "BenchmarkScoring": { "additionalProperties": false, "description": "How a release turns attempts into one score per configuration.\n\nRepeated attempts of a Task are averaged first. Task scores are then\ncombined with ``task_weights``; a Task without a weight counts once. Only the\nVerifier score on each Trial counts; step rewards reach it only through a\nVerifier whose check is ``total_reward``.\n\nAttributes:\n metric: Human-readable name of the success metric, such as\n ``\"Resolution rate\"``. It labels the Verifier score; it never\n selects a different value to rank on.\n description: What a score means, in words a reader can act on.\n aggregation: How Task scores combine. ``\"weighted_mean\"`` averages\n attempts per Task, then takes the ``task_weights`` mean.\n score_range: Lowest and highest possible Task score.\n success_threshold: A Task attempt at or above this score counts as a\n success. Left out, it is the top of ``score_range``. ``None`` means\n the Benchmark has no pass/fail notion.\n task_weights: Weight per Task name. Unlisted Tasks weigh 1.\n coverage_required: Weighted share of Tasks that must have a score\n before a result can be ranked officially.\n agent_failure: How an attempt that ended without a Verifier score\n because of the Agent (for example, a budget ran out) is scored.\n ``\"zero\"`` scores it at the bottom of the range.\n infrastructure_error: Attempts lost to the runtime or provider are\n excluded from scores and reported separately. So are attempts whose\n Verifier score falls outside ``score_range``.\n harness_timeout: How an attempt whose Harness ran past its time and was\n killed is scored. ``\"infrastructure_error\"`` excludes it;\n ``\"agent_failure\"`` scores it like a budget that ran out. A Harness\n is told its deadline in ``PLURAL_STOP_SECONDS``, and the built-in\n Harness stops itself before it.\n min_tasks_for_interval: Fewest scored Tasks before an uncertainty\n interval is reported. Below this, uncertainty is shown as unknown.", "properties": { "metric": { "default": "Score", "title": "Metric", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "aggregation": { "const": "weighted_mean", "default": "weighted_mean", "title": "Aggregation", "type": "string" }, "score_range": { "default": [ 0.0, 1.0 ], "maxItems": 2, "minItems": 2, "prefixItems": [ { "type": "number" }, { "type": "number" } ], "title": "Score Range", "type": "array" }, "success_threshold": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": 1.0, "title": "Success Threshold" }, "task_weights": { "additionalProperties": { "type": "number" }, "title": "Task Weights", "type": "object" }, "coverage_required": { "default": 1.0, "exclusiveMinimum": 0, "maximum": 1, "title": "Coverage Required", "type": "number" }, "agent_failure": { "default": "zero", "enum": [ "zero", "exclude" ], "title": "Agent Failure", "type": "string" }, "infrastructure_error": { "const": "exclude", "default": "exclude", "title": "Infrastructure Error", "type": "string" }, "harness_timeout": { "default": "infrastructure_error", "enum": [ "infrastructure_error", "agent_failure" ], "title": "Harness Timeout", "type": "string" }, "min_tasks_for_interval": { "default": 5, "minimum": 2, "title": "Min Tasks For Interval", "type": "integer" } }, "title": "BenchmarkScoring", "type": "object" }, "DeterministicVerifier": { "additionalProperties": false, "anyOf": [ { "required": [ "check" ] }, { "required": [ "command" ] }, { "required": [ "reward" ] }, { "required": [ "hindsight" ] } ], "description": "Code that verifies an episode: it scores the end, rewards actions, or both.\n\n``check`` scores the finished episode; that score is what Tasks and\nBenchmarks rank on. It is a function of the :class:`Episode`, a\n``{\"python\": \"verify.py:check\"}`` reference, or a command argv.\n\n``reward`` rewards each action while the episode runs. It is a function of\none :class:`Step`, or a ``{\"python\": \"verify.py:reward\"}`` reference, and\nruns in the Runtime beside the Apps, so it can read every App. What it\nreturns goes to the Trial's reward ledger. Rewards reach the score only\nwhen ``check`` is :func:`total_reward`.\n\n``hindsight`` credits the finished trajectory as a whole: a function of a\n:class:`~plural.rewards.HindsightTrial` that returns\n:class:`~plural.rewards.Credit` values, or a ``{\"python\":\n\"verify.py:efficiency\"}`` reference. It runs after the score, and can also\nrun later on stored Trials with ``plural verifier apply``.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "default": [], "items": { "$ref": "#/$defs/RubricCriterion" }, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "deterministic", "default": "deterministic", "title": "Kind", "type": "string" }, "check": { "default": null, "title": "Check" }, "reward": { "default": null, "title": "Reward" }, "hindsight": { "default": null, "title": "Hindsight" }, "runtime": { "$ref": "#/$defs/VerifierRuntime" }, "result_path": { "default": "verifier-result.json", "title": "Result Path", "type": "string" }, "evidence_required": { "default": true, "title": "Evidence Required", "type": "boolean" } }, "required": [ "name" ], "title": "DeterministicVerifier", "type": "object" }, "EnvironmentResource": { "additionalProperties": false, "description": "Data or application supplied by an Environment.\n\nThe short form ``Resource(\"policies/refunds.md\")`` stages one file from\nthe Environment's source package. The mapping form keeps its explicit\n``delivery``. ``Resource.from_file(path)`` reads a file of any type: text\ntravels inline, and anything else, such as a PDF or a drawing, as a\n``blob`` named by its digest.", "properties": { "kind": { "enum": [ "data", "application", "file" ], "title": "Kind", "type": "string" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "path": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Path" }, "uri": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Uri" }, "digest": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Digest" }, "content_type": { "default": "", "title": "Content Type", "type": "string" }, "config": { "additionalProperties": true, "title": "Config", "type": "object" }, "description": { "default": "", "title": "Description", "type": "string" }, "delivery": { "default": "descriptor", "enum": [ "descriptor", "source", "inline", "resolver", "blob" ], "title": "Delivery", "type": "string" }, "content": { "anyOf": [ { "maxLength": 1048576, "type": "string" }, { "type": "null" } ], "default": null, "title": "Content" }, "resolver": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Resolver" } }, "required": [ "kind", "name" ], "title": "EnvironmentResource", "type": "object" }, "EvaluationTrack": { "additionalProperties": false, "description": "Versioned rules that make results on one track comparable.\n\nA ``models`` track holds the setup fixed so results compare models: one\nharness, no extra instructions, and fixed inference settings. An\n``agents`` track compares complete systems, so harness, instructions,\ntools, and settings may differ within the declared limits.\n\nA track version is immutable within a Benchmark. Changing a rule needs a\nnew track version, and results on different versions are not ranked\ntogether.\n\nAttributes:\n id: Stable track identifier, such as ``\"controlled\"``.\n version: Semantic version of these rules.\n name: Label shown to readers.\n kind: ``\"models\"`` or ``\"agents\"``.\n description: What the track holds constant, in plain words.\n attempts: Attempts per Task every ranked result must have.\n max_retries: Infrastructure retries allowed per attempt.\n harnesses: Allowed harness names. Empty allows any on an ``agents``\n track and means the built-in loop on a ``models`` track.\n instructions: ``\"none\"`` forbids Agent instructions beyond the Task.\n temperature: Required temperature, when fixed.\n max_tokens: Required output-token limit, when fixed.\n tools: How tools are provided, in words; tools come from the Task\n Environment and the allowed harnesses.\n max_turns: Highest turn limit an Environment in this release may set.\n max_seconds: Highest time limit an Environment may set.\n max_cost_usd: Highest cost limit an Environment may set.", "properties": { "id": { "title": "Id", "type": "string" }, "version": { "default": "1.0.0", "title": "Version", "type": "string" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "kind": { "enum": [ "models", "agents" ], "title": "Kind", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "attempts": { "default": 1, "maximum": 32, "minimum": 1, "title": "Attempts", "type": "integer" }, "max_retries": { "default": 0, "maximum": 10, "minimum": 0, "title": "Max Retries", "type": "integer" }, "harnesses": { "default": [], "items": { "type": "string" }, "title": "Harnesses", "type": "array" }, "instructions": { "default": "any", "enum": [ "none", "any" ], "title": "Instructions", "type": "string" }, "temperature": { "anyOf": [ { "type": "number" }, { "type": "null" } ], "default": null, "title": "Temperature" }, "max_tokens": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Max Tokens" }, "tools": { "default": "Environment actions only.", "title": "Tools", "type": "string" }, "max_turns": { "anyOf": [ { "minimum": 1, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Max Turns" }, "max_seconds": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Max Seconds" }, "max_cost_usd": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Max Cost Usd" } }, "required": [ "id", "name", "kind" ], "title": "EvaluationTrack", "type": "object" }, "HumanVerifier": { "additionalProperties": false, "description": "Human review verifier.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "info": { "default": null, "title": "Info" }, "criteria": { "items": { "$ref": "#/$defs/RubricCriterion" }, "minItems": 1, "title": "Criteria", "type": "array" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "secrets": { "default": [], "items": { "$ref": "#/$defs/SecretReference" }, "title": "Secrets", "type": "array" }, "kind": { "const": "human", "default": "human", "title": "Kind", "type": "string" }, "instructions": { "default": "", "title": "Instructions", "type": "string" } }, "required": [ "name", "criteria" ], "title": "HumanVerifier", "type": "object" }, "NetworkMode": { "description": "Requested sandbox network policy.\n\nUse public, no-network, or allowlist to configure outbound access.", "enum": [ "public", "no-network", "allowlist", "public", "no-network", "allowlist" ], "title": "NetworkMode", "type": "string" }, "ResourceRequirements": { "additionalProperties": false, "description": "Optional compute limits.", "properties": { "cpu": { "anyOf": [ { "exclusiveMinimum": 0, "type": "number" }, { "type": "null" } ], "default": null, "title": "Cpu" }, "memory_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Memory Mb" }, "pids": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Pids" }, "disk_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Disk Mb" }, "storage_mb": { "anyOf": [ { "exclusiveMinimum": 0, "type": "integer" }, { "type": "null" } ], "default": null, "title": "Storage Mb" } }, "title": "ResourceRequirements", "type": "object" }, "RubricCriterion": { "additionalProperties": false, "description": "One Agent or Human rubric criterion.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "description": { "minLength": 1, "title": "Description", "type": "string" }, "weight": { "default": 1, "exclusiveMinimum": 0, "title": "Weight", "type": "number" }, "min_score": { "default": 0, "title": "Min Score", "type": "number" }, "max_score": { "default": 1, "title": "Max Score", "type": "number" } }, "required": [ "name", "description" ], "title": "RubricCriterion", "type": "object" }, "SecretReference": { "additionalProperties": false, "description": "A named value an App needs, which it never stores itself.\n\nIn an App's ``secrets`` it is a setup secret, such as a sandbox's admin\ntoken, supplied by the project for reset and snapshot hooks and Verifiers.\nIn its ``credentials`` it is supplied by the Agent that runs, such as the\nSalesforce token of the user that Agent acts as. Either way the value\nreaches only the App host, never the Harness or the model.", "properties": { "name": { "pattern": "^[A-Z_][A-Z0-9_]*$", "title": "Name", "type": "string" }, "required": { "default": true, "title": "Required", "type": "boolean" }, "description": { "default": "", "title": "Description", "type": "string" }, "scopes": { "default": [], "items": { "type": "string" }, "title": "Scopes", "type": "array" } }, "required": [ "name" ], "title": "SecretReference", "type": "object" }, "Task": { "additionalProperties": false, "description": "One versioned unit of work in a Python-authored Environment.\n\nIn an Environment with a party, several Agents share each episode, one\nper seat. Each gets ``instructions``, and then its seat's entry in\n``seat_instructions``, keyed ``seat_1``, ``seat_2``, and so on.\n\n``expected`` is the answer key: what a correct episode produces. Verifiers\nread it as ``episode.expected``; it never reaches the Agent, unlike ``info``\nand ``metadata``.", "properties": { "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "default": "0.1.0", "title": "Version", "type": "string" }, "instructions": { "minLength": 1, "title": "Instructions", "type": "string" }, "goals": { "default": [], "items": { "type": "string" }, "title": "Goals", "type": "array" }, "info": { "default": null, "title": "Info" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" }, "environment": { "title": "Environment" }, "verifiers": { "items": { "discriminator": { "mapping": { "agent": "#/$defs/AgentVerifier", "deterministic": "#/$defs/DeterministicVerifier", "human": "#/$defs/HumanVerifier" }, "propertyName": "kind" }, "oneOf": [ { "$ref": "#/$defs/DeterministicVerifier" }, { "$ref": "#/$defs/AgentVerifier" }, { "$ref": "#/$defs/HumanVerifier" } ] }, "minItems": 1, "title": "Verifiers", "type": "array" }, "resources": { "default": [], "items": { "$ref": "#/$defs/EnvironmentResource" }, "title": "Resources", "type": "array" }, "initial_state": { "additionalProperties": true, "title": "Initial State", "type": "object" }, "seat_instructions": { "additionalProperties": { "type": "string" }, "title": "Seat Instructions", "type": "object" }, "expected": { "default": null, "title": "Expected" } }, "required": [ "name", "instructions", "environment", "verifiers" ], "title": "Task", "type": "object" }, "VerifierRuntime": { "additionalProperties": false, "description": "Machine for a verifier that needs its own sandbox.\n\nThe default runs in the Task Environment. Set a provider, image, network,\nresource limit, or timeout to use a different machine.", "properties": { "provider": { "default": "docker", "minLength": 1, "title": "Provider", "type": "string" }, "image": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Image" }, "network": { "$ref": "#/$defs/NetworkMode", "default": "public" }, "allowed_hosts": { "default": [], "items": { "type": "string" }, "title": "Allowed Hosts", "type": "array" }, "compute": { "$ref": "#/$defs/ResourceRequirements" }, "timeout_seconds": { "default": 60, "exclusiveMinimum": 0, "title": "Timeout Seconds", "type": "number" } }, "title": "VerifierRuntime", "type": "object" } }, "additionalProperties": false, "description": "A semantic version that pins an ordered set of Tasks.\n\nEach version is a release. Beyond its Tasks it may declare categories,\nscoring rules, and evaluation tracks; see :class:`BenchmarkRelease`.", "properties": { "categories": { "default": [], "items": { "$ref": "#/$defs/BenchmarkCategory" }, "title": "Categories", "type": "array" }, "scoring": { "$ref": "#/$defs/BenchmarkScoring" }, "tracks": { "default": [], "items": { "$ref": "#/$defs/EvaluationTrack" }, "title": "Tracks", "type": "array" }, "default_view": { "default": "agents", "enum": [ "models", "agents" ], "title": "Default View", "type": "string" }, "purpose": { "default": "", "title": "Purpose", "type": "string" }, "success": { "default": "", "title": "Success", "type": "string" }, "limitations": { "default": [], "items": { "type": "string" }, "title": "Limitations", "type": "array" }, "license": { "default": "", "title": "License", "type": "string" }, "forked_from": { "anyOf": [ { "type": "string" }, { "type": "null" } ], "default": null, "title": "Forked From" }, "name": { "minLength": 1, "title": "Name", "type": "string" }, "version": { "title": "Version", "type": "string" }, "tasks": { "items": { "$ref": "#/$defs/Task" }, "minItems": 1, "title": "Tasks", "type": "array" }, "primary_metric": { "const": "score", "default": "score", "title": "Primary Metric", "type": "string" }, "description": { "default": "", "title": "Description", "type": "string" }, "metadata": { "additionalProperties": true, "title": "Metadata", "type": "object" } }, "required": [ "name", "version", "tasks" ], "title": "Benchmark", "type": "object" } }