import Runloop from '@runloop/api-client';
const client = new Runloop({
bearerToken: process.env['RUNLOOP_API_KEY'], // This is the default and can be omitted
});
// Automatically fetches more pages as needed.
for await (const scenarioView of client.scenarios.list()) {
console.log(scenarioView.id);
}{
"scenarios": [
{
"id": "<string>",
"name": "<string>",
"input_context": {
"problem_statement": "<string>",
"additional_context": {}
},
"scoring_contract": {
"scoring_function_parameters": [
{
"name": "<string>",
"scorer": {
"search_directory": "<string>",
"pattern": "<string>",
"type": "ast_grep_scorer",
"lang": "<string>"
},
"weight": 123
}
]
},
"metadata": {},
"environment": {
"blueprint_id": "<string>",
"snapshot_id": "<string>",
"launch_parameters": {
"launch_commands": [
"<string>"
],
"resource_size_request": "X_SMALL",
"keep_alive_time_seconds": 123,
"available_ports": [
123
],
"after_idle": {
"idle_time_seconds": 123,
"on_idle": "shutdown"
},
"custom_cpu_cores": 123,
"custom_gb_memory": 123,
"custom_disk_size": 123,
"architecture": "x86_64",
"user_parameters": {
"username": "<string>",
"uid": 123
},
"required_services": [
"<string>"
]
},
"working_directory": "<string>"
},
"reference_output": "<string>",
"required_environment_variables": [
"<string>"
],
"required_secret_names": [
"<string>"
],
"is_public": true,
"validation_type": "UNSPECIFIED"
}
],
"has_more": true,
"total_count": 123,
"remaining_count": 123
}List all Scenarios matching filter.
import Runloop from '@runloop/api-client';
const client = new Runloop({
bearerToken: process.env['RUNLOOP_API_KEY'], // This is the default and can be omitted
});
// Automatically fetches more pages as needed.
for await (const scenarioView of client.scenarios.list()) {
console.log(scenarioView.id);
}{
"scenarios": [
{
"id": "<string>",
"name": "<string>",
"input_context": {
"problem_statement": "<string>",
"additional_context": {}
},
"scoring_contract": {
"scoring_function_parameters": [
{
"name": "<string>",
"scorer": {
"search_directory": "<string>",
"pattern": "<string>",
"type": "ast_grep_scorer",
"lang": "<string>"
},
"weight": 123
}
]
},
"metadata": {},
"environment": {
"blueprint_id": "<string>",
"snapshot_id": "<string>",
"launch_parameters": {
"launch_commands": [
"<string>"
],
"resource_size_request": "X_SMALL",
"keep_alive_time_seconds": 123,
"available_ports": [
123
],
"after_idle": {
"idle_time_seconds": 123,
"on_idle": "shutdown"
},
"custom_cpu_cores": 123,
"custom_gb_memory": 123,
"custom_disk_size": 123,
"architecture": "x86_64",
"user_parameters": {
"username": "<string>",
"uid": 123
},
"required_services": [
"<string>"
]
},
"working_directory": "<string>"
},
"reference_output": "<string>",
"required_environment_variables": [
"<string>"
],
"required_secret_names": [
"<string>"
],
"is_public": true,
"validation_type": "UNSPECIFIED"
}
],
"has_more": true,
"total_count": 123,
"remaining_count": 123
}Bearer authentication header of the form Bearer <token>, where <token> is your auth token.
Query for Scenarios with a given name.
Filter scenarios by benchmark ID.
The limit of items to return. Default is 20.
Load the next page of data starting after the item with the given ID.
OK
List of Scenarios matching filter.
Show child attributes
The ID of the Scenario.
The name of the Scenario.
The scoring contract for the Scenario.
Show child attributes
A list of scoring functions used to evaluate the Scenario.
Show child attributes
Name of scoring function. Names must only contain [a-zA-Z0-9_-].
The scoring function to use for evaluating this scenario. The type field determines which built-in function to use.
Show child attributes
The path to search.
AST pattern to match. Pattern will be passed to ast-grep using the commandline surround by double quotes ("), so make sure to use proper escaping (for example, $$$).
ast_grep_scorer The language of the pattern.
Weight to apply to scoring function score. Weights of all scoring functions should sum to 1.0.
The Environment in which the Scenario is run.
Show child attributes
Use the blueprint with matching ID.
Use the snapshot with matching ID.
Optional launch parameters to apply to the devbox environment at launch.
Show child attributes
Set of commands to be run at launch time, before the entrypoint process is run.
Manual resource configuration for Devbox. If not set, defaults will be used.
X_SMALL, SMALL, MEDIUM, LARGE, X_LARGE, XX_LARGE, CUSTOM_SIZE Time in seconds after which Devbox will automatically shutdown. Default is 1 hour. Maximum is 48 hours (172800 seconds).
A list of ports to make available on the Devbox. Only ports made available will be surfaced to create tunnels via the 'createTunnel' API.
Configure Devbox lifecycle based on idle activity. If after_idle is set, Devbox will ignore keep_alive_time_seconds.
Show child attributes
After idle_time_seconds, on_idle action will be taken.
Action to take after Devbox becomes idle.
shutdown, suspend custom resource size, number of cpu cores, must be multiple of 2. Min is 1, max is 16.
custom memory size, number in GiB, must be a multiple of 2. Min is 2GiB, max is 64GiB.
custom disk size, number in GiB, must be a multiple of 2. Min is 2GiB, max is 64GiB.
The target architecture for the Devbox. If unset, defaults to x86_64.
x86_64, arm64 Specify the user for execution on Devbox. If not set, default user will be used.
A list of ContainerizedService names to be started when a Devbox is created. A valid ContainerizedService must be specified in Blueprint to be started.
The working directory where the agent is expected to fulfill the scenario. Scoring functions also run from the working directory.
A string representation of the reference output to solve the scenario. Commonly can be the result of a git diff or a sequence of command actions to apply to the environment.
Environment variables required to run the scenario. If any required environment variables are missing, the scenario will fail to start.
Environment variables required to run the scenario. If any required secrets are missing, the scenario will fail to start.
Whether this scenario is public.
Validation strategy.
UNSPECIFIED, FORWARD, REVERSE, EVALUATION Was this page helpful?