Skip to main content
POST
/
v1
/
scenarios
JavaScript
import Runloop from '@runloop/api-client';

const client = new Runloop({
  bearerToken: process.env['RUNLOOP_API_KEY'], // This is the default and can be omitted
});

const scenarioView = await client.scenarios.create({
  input_context: { problem_statement: 'problem_statement' },
  name: 'name',
  scoring_contract: {
    scoring_function_parameters: [
      {
        name: 'name',
        scorer: {
          pattern: 'pattern',
          search_directory: 'search_directory',
          type: 'ast_grep_scorer',
        },
        weight: 0,
      },
    ],
  },
});

console.log(scenarioView.id);
import os
from runloop_api_client import Runloop

client = Runloop(
    bearer_token=os.environ.get("RUNLOOP_API_KEY"),  # This is the default and can be omitted
)
scenario_view = client.scenarios.create(
    input_context={
        "problem_statement": "problem_statement"
    },
    name="name",
    scoring_contract={
        "scoring_function_parameters": [{
            "name": "name",
            "scorer": {
                "pattern": "pattern",
                "search_directory": "search_directory",
                "type": "ast_grep_scorer",
            },
            "weight": 0,
        }]
    },
)
print(scenario_view.id)
curl --request POST \
  --url https://api.runloop.ai/v1/scenarios \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "name": "<string>",
  "input_context": {
    "problem_statement": "<string>",
    "additional_context": {}
  },
  "scoring_contract": {
    "scoring_function_parameters": [
      {
        "name": "<string>",
        "scorer": {
          "search_directory": "<string>",
          "pattern": "<string>",
          "type": "ast_grep_scorer",
          "lang": "<string>"
        },
        "weight": 123
      }
    ]
  },
  "environment_parameters": {
    "blueprint_id": "<string>",
    "snapshot_id": "<string>",
    "launch_parameters": {
      "launch_commands": [
        "<string>"
      ],
      "available_ports": [
        123
      ],
      "keep_alive_time_seconds": 123,
      "after_idle": {
        "idle_time_seconds": 123
      },
      "custom_cpu_cores": 123,
      "custom_gb_memory": 123,
      "custom_disk_size": 123,
      "user_parameters": {
        "username": "<string>",
        "uid": 123
      },
      "required_services": [
        "<string>"
      ],
      "network_policy_id": "<string>",
      "lifecycle": {
        "after_idle": {
          "idle_time_seconds": 123
        },
        "resume_triggers": {
          "http": true,
          "axon_event": true
        },
        "lifecycle_hooks": {
          "suspend_commands": [
            "<string>"
          ],
          "suspend_deadline_ms": 123
        }
      }
    },
    "working_directory": "<string>"
  },
  "metadata": {},
  "reference_output": "<string>",
  "required_environment_variables": [
    "<string>"
  ],
  "required_secret_names": [
    "<string>"
  ],
  "scorer_timeout_sec": 123
}
'
<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.runloop.ai/v1/scenarios",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_ENCODING => "",
  CURLOPT_MAXREDIRS => 10,
  CURLOPT_TIMEOUT => 30,
  CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode([
    'name' => '<string>',
    'input_context' => [
        'problem_statement' => '<string>',
        'additional_context' => [
                
        ]
    ],
    'scoring_contract' => [
        'scoring_function_parameters' => [
                [
                                'name' => '<string>',
                                'scorer' => [
                                                                'search_directory' => '<string>',
                                                                'pattern' => '<string>',
                                                                'type' => 'ast_grep_scorer',
                                                                'lang' => '<string>'
                                ],
                                'weight' => 123
                ]
        ]
    ],
    'environment_parameters' => [
        'blueprint_id' => '<string>',
        'snapshot_id' => '<string>',
        'launch_parameters' => [
                'launch_commands' => [
                                '<string>'
                ],
                'available_ports' => [
                                123
                ],
                'keep_alive_time_seconds' => 123,
                'after_idle' => [
                                'idle_time_seconds' => 123
                ],
                'custom_cpu_cores' => 123,
                'custom_gb_memory' => 123,
                'custom_disk_size' => 123,
                'user_parameters' => [
                                'username' => '<string>',
                                'uid' => 123
                ],
                'required_services' => [
                                '<string>'
                ],
                'network_policy_id' => '<string>',
                'lifecycle' => [
                                'after_idle' => [
                                                                'idle_time_seconds' => 123
                                ],
                                'resume_triggers' => [
                                                                'http' => true,
                                                                'axon_event' => true
                                ],
                                'lifecycle_hooks' => [
                                                                'suspend_commands' => [
                                                                                                                                '<string>'
                                                                ],
                                                                'suspend_deadline_ms' => 123
                                ]
                ]
        ],
        'working_directory' => '<string>'
    ],
    'metadata' => [
        
    ],
    'reference_output' => '<string>',
    'required_environment_variables' => [
        '<string>'
    ],
    'required_secret_names' => [
        '<string>'
    ],
    'scorer_timeout_sec' => 123
  ]),
  CURLOPT_HTTPHEADER => [
    "Authorization: Bearer <token>",
    "Content-Type: application/json"
  ],
]);

$response = curl_exec($curl);
$err = curl_error($curl);

curl_close($curl);

if ($err) {
  echo "cURL Error #:" . $err;
} else {
  echo $response;
}
package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.runloop.ai/v1/scenarios"

	payload := strings.NewReader("{\n  \"name\": \"<string>\",\n  \"input_context\": {\n    \"problem_statement\": \"<string>\",\n    \"additional_context\": {}\n  },\n  \"scoring_contract\": {\n    \"scoring_function_parameters\": [\n      {\n        \"name\": \"<string>\",\n        \"scorer\": {\n          \"search_directory\": \"<string>\",\n          \"pattern\": \"<string>\",\n          \"type\": \"ast_grep_scorer\",\n          \"lang\": \"<string>\"\n        },\n        \"weight\": 123\n      }\n    ]\n  },\n  \"environment_parameters\": {\n    \"blueprint_id\": \"<string>\",\n    \"snapshot_id\": \"<string>\",\n    \"launch_parameters\": {\n      \"launch_commands\": [\n        \"<string>\"\n      ],\n      \"available_ports\": [\n        123\n      ],\n      \"keep_alive_time_seconds\": 123,\n      \"after_idle\": {\n        \"idle_time_seconds\": 123\n      },\n      \"custom_cpu_cores\": 123,\n      \"custom_gb_memory\": 123,\n      \"custom_disk_size\": 123,\n      \"user_parameters\": {\n        \"username\": \"<string>\",\n        \"uid\": 123\n      },\n      \"required_services\": [\n        \"<string>\"\n      ],\n      \"network_policy_id\": \"<string>\",\n      \"lifecycle\": {\n        \"after_idle\": {\n          \"idle_time_seconds\": 123\n        },\n        \"resume_triggers\": {\n          \"http\": true,\n          \"axon_event\": true\n        },\n        \"lifecycle_hooks\": {\n          \"suspend_commands\": [\n            \"<string>\"\n          ],\n          \"suspend_deadline_ms\": 123\n        }\n      }\n    },\n    \"working_directory\": \"<string>\"\n  },\n  \"metadata\": {},\n  \"reference_output\": \"<string>\",\n  \"required_environment_variables\": [\n    \"<string>\"\n  ],\n  \"required_secret_names\": [\n    \"<string>\"\n  ],\n  \"scorer_timeout_sec\": 123\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(string(body))

}
HttpResponse<String> response = Unirest.post("https://api.runloop.ai/v1/scenarios")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"name\": \"<string>\",\n  \"input_context\": {\n    \"problem_statement\": \"<string>\",\n    \"additional_context\": {}\n  },\n  \"scoring_contract\": {\n    \"scoring_function_parameters\": [\n      {\n        \"name\": \"<string>\",\n        \"scorer\": {\n          \"search_directory\": \"<string>\",\n          \"pattern\": \"<string>\",\n          \"type\": \"ast_grep_scorer\",\n          \"lang\": \"<string>\"\n        },\n        \"weight\": 123\n      }\n    ]\n  },\n  \"environment_parameters\": {\n    \"blueprint_id\": \"<string>\",\n    \"snapshot_id\": \"<string>\",\n    \"launch_parameters\": {\n      \"launch_commands\": [\n        \"<string>\"\n      ],\n      \"available_ports\": [\n        123\n      ],\n      \"keep_alive_time_seconds\": 123,\n      \"after_idle\": {\n        \"idle_time_seconds\": 123\n      },\n      \"custom_cpu_cores\": 123,\n      \"custom_gb_memory\": 123,\n      \"custom_disk_size\": 123,\n      \"user_parameters\": {\n        \"username\": \"<string>\",\n        \"uid\": 123\n      },\n      \"required_services\": [\n        \"<string>\"\n      ],\n      \"network_policy_id\": \"<string>\",\n      \"lifecycle\": {\n        \"after_idle\": {\n          \"idle_time_seconds\": 123\n        },\n        \"resume_triggers\": {\n          \"http\": true,\n          \"axon_event\": true\n        },\n        \"lifecycle_hooks\": {\n          \"suspend_commands\": [\n            \"<string>\"\n          ],\n          \"suspend_deadline_ms\": 123\n        }\n      }\n    },\n    \"working_directory\": \"<string>\"\n  },\n  \"metadata\": {},\n  \"reference_output\": \"<string>\",\n  \"required_environment_variables\": [\n    \"<string>\"\n  ],\n  \"required_secret_names\": [\n    \"<string>\"\n  ],\n  \"scorer_timeout_sec\": 123\n}")
  .asString();
require 'uri'
require 'net/http'

url = URI("https://api.runloop.ai/v1/scenarios")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"name\": \"<string>\",\n  \"input_context\": {\n    \"problem_statement\": \"<string>\",\n    \"additional_context\": {}\n  },\n  \"scoring_contract\": {\n    \"scoring_function_parameters\": [\n      {\n        \"name\": \"<string>\",\n        \"scorer\": {\n          \"search_directory\": \"<string>\",\n          \"pattern\": \"<string>\",\n          \"type\": \"ast_grep_scorer\",\n          \"lang\": \"<string>\"\n        },\n        \"weight\": 123\n      }\n    ]\n  },\n  \"environment_parameters\": {\n    \"blueprint_id\": \"<string>\",\n    \"snapshot_id\": \"<string>\",\n    \"launch_parameters\": {\n      \"launch_commands\": [\n        \"<string>\"\n      ],\n      \"available_ports\": [\n        123\n      ],\n      \"keep_alive_time_seconds\": 123,\n      \"after_idle\": {\n        \"idle_time_seconds\": 123\n      },\n      \"custom_cpu_cores\": 123,\n      \"custom_gb_memory\": 123,\n      \"custom_disk_size\": 123,\n      \"user_parameters\": {\n        \"username\": \"<string>\",\n        \"uid\": 123\n      },\n      \"required_services\": [\n        \"<string>\"\n      ],\n      \"network_policy_id\": \"<string>\",\n      \"lifecycle\": {\n        \"after_idle\": {\n          \"idle_time_seconds\": 123\n        },\n        \"resume_triggers\": {\n          \"http\": true,\n          \"axon_event\": true\n        },\n        \"lifecycle_hooks\": {\n          \"suspend_commands\": [\n            \"<string>\"\n          ],\n          \"suspend_deadline_ms\": 123\n        }\n      }\n    },\n    \"working_directory\": \"<string>\"\n  },\n  \"metadata\": {},\n  \"reference_output\": \"<string>\",\n  \"required_environment_variables\": [\n    \"<string>\"\n  ],\n  \"required_secret_names\": [\n    \"<string>\"\n  ],\n  \"scorer_timeout_sec\": 123\n}"

response = http.request(request)
puts response.read_body
{
  "id": "<string>",
  "name": "<string>",
  "input_context": {
    "problem_statement": "<string>",
    "additional_context": {}
  },
  "scoring_contract": {
    "scoring_function_parameters": [
      {
        "name": "<string>",
        "scorer": {
          "search_directory": "<string>",
          "pattern": "<string>",
          "type": "ast_grep_scorer",
          "lang": "<string>"
        },
        "weight": 123
      }
    ]
  },
  "metadata": {},
  "environment": {
    "blueprint_id": "<string>",
    "snapshot_id": "<string>",
    "launch_parameters": {
      "launch_commands": [
        "<string>"
      ],
      "available_ports": [
        123
      ],
      "keep_alive_time_seconds": 123,
      "after_idle": {
        "idle_time_seconds": 123
      },
      "custom_cpu_cores": 123,
      "custom_gb_memory": 123,
      "custom_disk_size": 123,
      "user_parameters": {
        "username": "<string>",
        "uid": 123
      },
      "required_services": [
        "<string>"
      ],
      "network_policy_id": "<string>",
      "lifecycle": {
        "after_idle": {
          "idle_time_seconds": 123
        },
        "resume_triggers": {
          "http": true,
          "axon_event": true
        },
        "lifecycle_hooks": {
          "suspend_commands": [
            "<string>"
          ],
          "suspend_deadline_ms": 123
        }
      }
    },
    "working_directory": "<string>"
  },
  "reference_output": "<string>",
  "required_environment_variables": [
    "<string>"
  ],
  "required_secret_names": [
    "<string>"
  ],
  "is_public": true,
  "scorer_timeout_sec": 123
}

Authorizations

​
Authorization
string
header
required

Bearer authentication header of the form Bearer <token>, where <token> is your auth token.

Body

application/json
​
name
string
required

Name of the scenario.

​
input_context
object
required

The input context for the Scenario.

​
scoring_contract
object
required

The scoring contract for the Scenario.

​
environment_parameters
object | null

The Environment in which the Scenario will run.

​
metadata
object | null

User defined metadata to attach to the scenario for organization.

​
reference_output
string | null

A string representation of the reference output to solve the scenario. Commonly can be the result of a git diff or a sequence of command actions to apply to the environment.

​
required_environment_variables
string[] | null

Environment variables required to run the scenario. If these variables are not provided, the scenario will fail to start.

​
required_secret_names
string[] | null

Secrets required to run the scenario (user secret name to scenario required secret name). If these secrets are not provided or the mapping is incorrect, the scenario will fail to start.

​
validation_type
enum<string> | null

Validation strategy.

Available options:
UNSPECIFIED,
FORWARD,
REVERSE,
EVALUATION
​
scorer_timeout_sec
integer<int32> | null

Timeout for scoring in seconds. Default 30 minutes (1800s).

Response

200 - application/json

OK

A ScenarioDefinitionView represents a repeatable AI coding evaluation test, complete with initial environment and scoring contract.

​
id
string
required

The ID of the Scenario.

​
name
string
required

The name of the Scenario.

​
input_context
object
required

The input context for the Scenario.

​
scoring_contract
object
required

The scoring contract for the Scenario.

​
metadata
object
required

User defined metadata to attach to the scenario for organization.

​
status
enum<string>
required

Whether the scenario is active or archived. Archived scenarios are excluded from listings and cannot be updated.

Available options:
active,
archived
​
environment
object | null

The Environment in which the Scenario is run.

​
reference_output
string | null

A string representation of the reference output to solve the scenario. Commonly can be the result of a git diff or a sequence of command actions to apply to the environment.

​
required_environment_variables
string[]

Environment variables required to run the scenario. If any required environment variables are missing, the scenario will fail to start.

​
required_secret_names
string[]

Environment variables required to run the scenario. If any required secrets are missing, the scenario will fail to start.

​
is_public
boolean

Whether this scenario is public.

​
validation_type
enum<string> | null

Validation strategy.

Available options:
UNSPECIFIED,
FORWARD,
REVERSE,
EVALUATION
​
scorer_timeout_sec
integer<int32> | null

Timeout for scoring in seconds. Default 30 minutes (1800s).