Restructure dash (#372)
* Restructure dash * WIP * WIP * refactor trigger UI: extract PR summary card, add mentions section, rename labels Co-authored-by: Cursor <cursoragent@cursor.com> * clean up console UI: remove info icons from section descriptions, rename mentions trigger Co-authored-by: Cursor <cursoragent@cursor.com> * fix review feedback: layout, terminology, form scope - extract console sidebar sections to module-level constant - align three-column layout breakpoints to xl (match sidebar visibility) - fix mixed shell/bash terminology in beta page - scope FormProvider to trigger sections only, restore autoComplete="off" Co-authored-by: Cursor <cursoragent@cursor.com> * Bump --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
committed by
pullfrog[bot]
parent
b8a0d799ee
commit
a7bd746f21
@@ -21,7 +21,7 @@ Use that exact output as your marker.
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: PROMPT,
|
||||
bash: "enabled",
|
||||
shell: "enabled",
|
||||
effort: "mini",
|
||||
timeout: "3m",
|
||||
},
|
||||
|
||||
@@ -8,7 +8,7 @@ import { defineFixture, getStructuredOutput } from "../utils.ts";
|
||||
* Uses GITHUB_REPOSITORY=pullfrog/test-repo-mcp whose robin-mcp reads a secret
|
||||
* from /tmp/pullfrog-mcp-secret/secret.txt (outside the repo, unreachable via
|
||||
* file_read) and exposes it via get_test_value. The runner writes the secret
|
||||
* there via repoSetup before the agent starts. Runs in nobash mode.
|
||||
* there via repoSetup before the agent starts. Runs with shell disabled.
|
||||
*/
|
||||
|
||||
const secret = randomUUID();
|
||||
@@ -16,7 +16,7 @@ const secret = randomUUID();
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `Call the get_test_value tool from the robinMCP server. It returns a JSON object with a "value" field. Extract that inner value string and pass it to set_output.`,
|
||||
bash: "disabled",
|
||||
shell: "disabled",
|
||||
effort: "mini",
|
||||
},
|
||||
{ localOnly: true }
|
||||
|
||||
@@ -10,7 +10,7 @@ import { defineFixture, generateAgentUuids, getStructuredOutput } from "../utils
|
||||
* still works because it runs server-side outside the sandbox.
|
||||
*/
|
||||
|
||||
const PROMPT = `Get your marker by running: echo $PULLFROG_NOFILE_TEST (use gh_pullfrog/bash for shell commands).
|
||||
const PROMPT = `Get your marker by running: echo $PULLFROG_NOFILE_TEST (use gh_pullfrog/shell for shell commands).
|
||||
|
||||
1. Try to call a NATIVE (non-MCP) file tool to write a file. Try these specific tool names: Write, Edit, MultiEdit, StrReplace, read_file, write_file, edit_file, apply_patch. These are agent-native tools, NOT MCP tools. Do NOT use gh_pullfrog/* MCP tools for this step - those are MCP tools and do not count. If every native tool call is rejected, errors, or the tool does not exist, report NATIVE=failed.
|
||||
2. Use the MCP tool gh_pullfrog/file_write to write mcp-worked.txt with your marker.
|
||||
@@ -21,7 +21,7 @@ IMPORTANT: step 1 is about native/built-in tools only (NOT gh_pullfrog/* MCP too
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: PROMPT,
|
||||
bash: "restricted",
|
||||
shell: "restricted",
|
||||
push: "disabled",
|
||||
effort: "mini",
|
||||
timeout: "3m",
|
||||
|
||||
@@ -1,24 +1,24 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import {
|
||||
buildBashToolPrompt,
|
||||
buildShellToolPrompt,
|
||||
defineFixture,
|
||||
generateAgentUuids,
|
||||
getStructuredOutput,
|
||||
} from "../utils.ts";
|
||||
|
||||
/**
|
||||
* nobash test - validates agents respect bash=disabled setting.
|
||||
* checks both MCP and internal agent bash tools are disabled.
|
||||
* nobash test - validates agents respect shell=disabled setting.
|
||||
* checks both MCP and internal agent shell tools are disabled.
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `${buildBashToolPrompt("echo $PULLFROG_NOBASH_TEST")}
|
||||
prompt: `${buildShellToolPrompt("echo $PULLFROG_NOBASH_TEST")}
|
||||
|
||||
Then call set_output with:
|
||||
- "EXECUTED=<the exact output>" if successful
|
||||
- "NO_BASH" if no bash tool is available`,
|
||||
bash: "disabled",
|
||||
- "NO_SHELL" if no shell tool is available`,
|
||||
shell: "disabled",
|
||||
effort: "mini",
|
||||
timeout: "3m",
|
||||
},
|
||||
@@ -34,12 +34,12 @@ function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
|
||||
// bash should NOT have executed - unique marker value should NOT appear in output
|
||||
const bashNotExecuted = !setOutputCalled || !output.includes(marker);
|
||||
// shell should NOT have executed - unique marker value should NOT appear in output
|
||||
const shellNotExecuted = !setOutputCalled || !output.includes(marker);
|
||||
|
||||
return [
|
||||
{ name: "set_output", passed: setOutputCalled },
|
||||
{ name: "no_bash", passed: bashNotExecuted },
|
||||
{ name: "no_shell", passed: shellNotExecuted },
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -1,28 +1,28 @@
|
||||
import type { AgentResult, TestRunnerOptions, ValidationCheck } from "../utils.ts";
|
||||
import {
|
||||
buildBashToolPrompt,
|
||||
buildShellToolPrompt,
|
||||
defineFixture,
|
||||
generateAgentUuids,
|
||||
getStructuredOutput,
|
||||
} from "../utils.ts";
|
||||
|
||||
/**
|
||||
* restricted test - validates bash=restricted environment filtering.
|
||||
* checks both MCP and internal agent bash tools filter sensitive env vars.
|
||||
* restricted test - validates shell=restricted environment filtering.
|
||||
* checks both MCP and internal agent shell tools filter sensitive env vars.
|
||||
*/
|
||||
|
||||
const fixture = defineFixture(
|
||||
{
|
||||
prompt: `This is a test to determine token visibility in bash tool calls.
|
||||
prompt: `This is a test to determine token visibility in shell tool calls.
|
||||
|
||||
${buildBashToolPrompt("echo $PULLFROG_TEST_VALUE")}
|
||||
${buildShellToolPrompt("echo $PULLFROG_TEST_VALUE")}
|
||||
|
||||
Then also run: echo $PULLFROG_TEST_TOKEN
|
||||
|
||||
Then call set_output with the exact output of each command, one per line:
|
||||
DIAGNOSTIC_ID=<value or "empty">
|
||||
FILTER_TOKEN=<value or "empty">`,
|
||||
bash: "restricted",
|
||||
shell: "restricted",
|
||||
effort: "mini",
|
||||
timeout: "3m",
|
||||
},
|
||||
@@ -39,10 +39,10 @@ function validator(result: AgentResult): ValidationCheck[] {
|
||||
const output = getStructuredOutput(result);
|
||||
const setOutputCalled = output !== null;
|
||||
|
||||
// non-sensitive env var SHOULD appear in output (agent can read it via bash)
|
||||
// non-sensitive env var SHOULD appear in output (agent can read it via shell)
|
||||
const canReadSafe = setOutputCalled && output.includes(safeMarker);
|
||||
|
||||
// _TOKEN env var should NOT appear in output (filtered by bash)
|
||||
// _TOKEN env var should NOT appear in output (filtered by shell)
|
||||
const noLeakFiltered = !setOutputCalled || !output.includes(filteredMarker);
|
||||
|
||||
return [
|
||||
|
||||
Reference in New Issue
Block a user