What is Runloop?

import Runloop from '@runloop/api-client';
import { generateText, tool } from 'ai';

// Create an isolated Devbox for the agent to use
const devbox = await runloopClient.devboxes.createAndAwaitRunning();
// Get the Runloop Tool Representation for the Devbox and convert them to Vercel AI Tools
const runloopDevboxShellTools = runloopClient.devboxes.tools.shellTools(devbox.id)      
const runloopDevboxFileTools = runloopClient.devboxes.tools.fileTools(devbox.id)      

// Use VercelAI SDK to create a simple agent that uses the Devbox to code a game
const { text: answer } = await generateText({
    model: openai('gpt-4o-2024-08-06'),
    tools: {
        ...runloopDevboxShellTools,
        ...runloopDevboxFileTools
    },
    maxSteps: 10,
    system:
        'You are an expert python coder that specializes in making CLI games.'
    prompt:
        'Create a CLI game that is a guessing game where the user has to guess a number between 1 and 100. Write the python script in the file `game.py`. The program should be callable from the command line via `python game.py`. Once you have generated the program, run it and print the output to stdout.'
  });

console.log(`ANSWER: ${answer}`);

import Runloop from '@runloop/api-client';

const runloopClient = new Runloop({
    bearerToken: process.env.RUNLOOP_API_KEY,
});

// First create a repository connection for Runloop to preindex the codebase we will run on:
const repositoryConnectionView = await client.repositories.createAndAwaitIndexing({ name: 'repository-name', owner: 'repository-owner' });

// Now let's use the repository connection to recreate the repository map heuristic:
// 1. We list all the code files in the repository
const codeFiles = await client.repositories.codeFiles(repositoryConnectionView.id);
// 2. We can use the special file viewer and query syntax to only view the files with method signatures
const files_with_method_signatures_only = await client.repositories.fileViewer(repositoryConnectionView.id, {
    files: codeFiles,
    // We make our query against the AST such that we only get class and method nodes and only include the signature and comments 
    query: 'class(signatures, comments) || method(signatures, comments)'
});

// Or we can simply use the built in repository map heuristic:
const repositoryMap = await client.repositories.repositoryMap(repositoryConnectionView.id);

import Runloop from '@runloop/api-client';

const runloopClient = new Runloop({
  bearerToken: process.env.RUNLOOP_API_KEY,
});

// 1. Create new Benchmarks or use existing ones like SWE-bench
const myBenchmark = await runloop.benchmarks.create({
    benchmark: 'UI Component Generation',
    testCases: [
        {
            name: 'Create a Login Page',
            problemStatement: 'Create a login page that allows users to login to the application. Call the component `LoginPage` and export it from the file `src/components/LoginPage.tsx`.',
            // Configure specific starting Devbox environemnts for your Agent to use as part of a benchmark test
            environment: 'DEFAULT_DEVBOX',
            outputContractRules: [
                {
                    type: 'typescript',
                    files: ["expected_snapshot.png"]
                    validate: (output) => {
                        // Validate the storybook snapshot is updated and roughly matches the expected_snapshot.png
                    }
                }
            ]                
        }
    ]
});

// 2. Run your agent against the benchmark
const benchmarkRun = await runloop.benchmarks.beginTestRun(myBenchmark.id);
// For each test case run our agent and report the completion
for (const testCase of myBenchmark.testCases) {
    const agentOutput = await myAgent.run({
        prompt: testCase.problemStatement,
        devbox: testCase.devbox
    });
    await runloop.benchmarks.reportTestCaseRun(testCaseRun.id, {
        output: agentOutput
    });
}