> ## Documentation Index
> Fetch the complete documentation index at: https://docs.devin.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Run Nightly E2E Tests on Staging

> Use the Devin automations API to run your end-to-end test suite against staging every night, file Linear tickets for failures, and post results to Slack.

export const UseCaseHero = ({title, description, prompt, category, features, devinUrl, agent, intent, playbookId, type}) => {
  const encodedPrompt = encodeURIComponent(prompt || '');
  const tag = 'docs-use-case-gallery';
  const utm = 'utm_source=docs&utm_medium=use-case-gallery&utm_campaign=hero-cta';
  const agentParams = (agent ? '&agent=' + agent : '') + (intent ? '&intent=' + intent : '') + (playbookId ? '&playbookId=' + playbookId : '');
  const devinHref = type === 'schedule' ? 'https://app.devin.ai/settings/schedules/create?' + utm + agentParams + (prompt ? '&prompt=' + encodedPrompt : '') : type === 'review' ? 'https://app.devin.ai/review?' + utm : agent === 'ada' ? 'https://app.devin.ai/search?' + utm + '&noSubmit=true' + (prompt ? '&prompt=' + encodedPrompt : '') : devinUrl ? devinUrl.includes('?') ? devinUrl + '&' + utm + agentParams : devinUrl + '?' + utm + agentParams : prompt ? 'https://app.devin.ai/?tags=' + tag + '&' + utm + agentParams + '&prompt=' + encodedPrompt : 'https://app.devin.ai/?' + utm + agentParams;
  const buttonLabel = type === 'schedule' ? 'Schedule in Devin ↗' : type === 'review' ? 'Set Up Devin Review ↗' : agent === 'advanced' ? 'Try in Devin ↗' : agent === 'dana' ? 'Try in Dana ↗' : agent === 'ada' ? 'Try in Ask Devin ↗' : 'Try in Devin ↗';
  const featureList = features ? features.split(',').map(f => f.trim()) : [];
  return <div className="uc-hero">
      <div className="uc-hero-inner">
        <div className="uc-hero-left">
          <h1 className="uc-hero-title">{title}</h1>
          <p className="uc-hero-desc">{description}</p>
          <div>
            <a href={devinHref} target="_blank" rel="noopener noreferrer" className="try-in-devin-btn">
              {buttonLabel}
            </a>
          </div>
        </div>
        <div className="uc-hero-meta">
          <div className="uc-meta-item">
            <span className="uc-meta-label">Author</span>
            <span className="uc-meta-value">Cognition</span>
          </div>
          <div className="uc-meta-item">
            <span className="uc-meta-label">Category</span>
            <span className="uc-meta-value">{category}</span>
          </div>
          {featureList.length > 0 && <div className="uc-meta-item">
              <span className="uc-meta-label">Features</span>
              <span className="uc-meta-value">{featureList.join(', ')}</span>
            </div>}
        </div>
      </div>
    </div>;
};

export const PromptBlock = ({children, type, agent, intent, playbookId}) => {
  var utm = 'utm_source=docs&utm_medium=use-case-gallery&utm_campaign=prompt-block';
  var tag = 'docs-use-case-gallery';
  var agentParams = (agent ? '&agent=' + agent : '') + (intent ? '&intent=' + intent : '') + (playbookId ? '&playbookId=' + playbookId : '');
  var label = type === 'schedule' ? 'Schedule in Devin' : type === 'playbook' ? 'Create Playbook' : type === 'knowledge' ? 'Add to Knowledge' : agent === 'advanced' ? 'Try in Devin' : agent === 'dana' ? 'Try in Dana' : agent === 'ada' ? 'Try in Ask Devin' : 'Try in Devin';
  var buildUrl = function (text) {
    var encoded = encodeURIComponent(text);
    if (type === 'schedule') return 'https://app.devin.ai/settings/schedules/create?' + utm + agentParams + '&prompt=' + encoded;
    if (type === 'playbook') return 'https://app.devin.ai/settings/playbooks/create?' + utm + '&body=' + encoded;
    if (type === 'knowledge') return 'https://app.devin.ai/knowledge?' + utm + '&body=' + encoded;
    if (agent === 'ada') return 'https://app.devin.ai/search?' + utm + '&noSubmit=true&prompt=' + encoded;
    return 'https://app.devin.ai/?tags=' + tag + '&' + utm + agentParams + '&prompt=' + encoded;
  };
  const ref = React.useRef(null);
  const [href, setHref] = React.useState('#');
  React.useEffect(() => {
    if (!ref.current) return;
    var codeEl = ref.current.querySelector('pre code');
    if (codeEl) {
      var text = codeEl.textContent.trim();
      if (text) setHref(buildUrl(text));
    }
    var header = ref.current.querySelector('[data-component-part="code-block-header"]');
    if (header && !header.querySelector('.prompt-block-devin-link')) {
      var link = document.createElement('a');
      link.href = href;
      link.target = '_blank';
      link.rel = 'noopener noreferrer';
      link.className = 'prompt-block-devin-link';
      link.style.cssText = 'display:inline-flex;align-items:center;gap:6px;text-decoration:none;color:#fff;font-size:11px;font-weight:500;padding:4px 10px;border-radius:6px;white-space:nowrap;background:#317CFF;transition:background 0.2s;margin-left:8px;';
      link.innerHTML = '<svg xmlns="http://www.w3.org/2000/svg" width="12" height="12" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 13v6a2 2 0 0 1-2 2H5a2 2 0 0 1-2-2V8a2 2 0 0 1 2-2h6"/><polyline points="15 3 21 3 21 9"/><line x1="10" y1="14" x2="21" y2="3"/></svg> ' + label;
      link.onmouseenter = function () {
        link.style.background = '#2968D9';
      };
      link.onmouseleave = function () {
        link.style.background = '#317CFF';
      };
      header.appendChild(link);
    }
    var existingLink = ref.current.querySelector('.prompt-block-devin-link');
    if (existingLink && href !== '#') existingLink.href = href;
  });
  return <div className="prompt-block" ref={ref}>{children}</div>;
};

<UseCaseHero title="Run Nightly E2E Tests on Staging" description="Schedule Devin to run your end-to-end test suite against staging every night and file tickets for failures." prompt="Help me set up a nightly Devin automation that runs our E2E test suite against staging. Follow the guide at https://docs.devin.ai/use-cases/gallery/api-scheduled-sessions and walk me through each step: creating a service user, writing the test playbook, and creating the automation via the API." category="Automations" features="API, Schedules, Playbooks" />

<div className="uc-detail-wrapper">
  <Tip>Don't want to set this up manually? Paste a link to this page into a Devin session and ask it to set everything up for you.</Tip>

  This guide walks through managing scheduled [automations](/product-guides/automations) via the [v3 API](/api-reference/v3/overview), which is useful for infrastructure-as-code workflows. You can also create and manage them directly on the [Automations page](https://app.devin.ai/automations?utm_source=docs\&utm_medium=use-case-gallery) without any API setup.

  <Steps>
    <Step title="Set up a service user for API access">
      Automations created through the API need a service user with the right permissions. You'll set one up once, then use its API key in all the calls below.

      1. Go to [app.devin.ai](https://app.devin.ai/?utm_source=docs\&utm_medium=use-case-gallery) > **Settings** > **Devin API**, open the **Service users** tab, and click **Provision service user**
      2. Assign a role that includes the `ManageOrgAutomations` permission
      3. Save the API key shown after provisioning — it's only displayed once and you'll use it as your `Bearer` token

      Your **organization ID** is shown at the top of the **Settings** > **Devin API** page. Enterprise service users whose role includes `ViewOrganizations` can also call the [List Organizations](/api-reference/v3/organizations/organizations) endpoint with their token:

      ```bash theme={null}
      curl "https://api.devin.ai/v3/enterprise/organizations" \
        -H "Authorization: Bearer $DEVIN_API_KEY"
      ```

      Export both values so the commands in this guide work as-is:

      ```bash theme={null}
      export DEVIN_API_KEY="sk-your-service-user-key"
      export ORG_ID="your-org-id"
      ```

      See the [API authentication docs](/api-reference/authentication) for more on service users and permissions.
    </Step>

    <Step title="Write a playbook for the test run">
      Before creating the automation, write a [playbook](/product-guides/creating-playbooks) that tells Devin exactly how to run your E2E suite and what to do with the results. Go to [**Settings > Playbooks**](https://app.devin.ai/settings/playbooks?utm_source=docs\&utm_medium=use-case-gallery) and create a new playbook — or ask Devin to generate one for you from a description of your test workflow. Here's an example for a Playwright suite:

      <PromptBlock type="playbook">
        ```txt Nightly E2E test playbook theme={null}
        You are running the nightly E2E test suite for acme/web-app against staging.

        Steps:
        1. Run the Playwright E2E suite targeting staging:
           `PLAYWRIGHT_BASE_URL=https://staging.acme.dev npx playwright test --reporter=json`
        2. Parse the JSON results. For each failing test:
           - Create a Linear ticket in the "QA" team with:
             - Title: "[Nightly E2E] <test name>"
             - Description: the full error message, stack trace, and the test file path
             - Label: "nightly-e2e-failure"
             - Priority: High if the test is in tests/critical/*, otherwise Medium
           - Attach any failure screenshots from the Playwright report
        3. If all tests pass, post a summary to the #qa-results Slack channel:
           "Nightly E2E: all <N> tests passed against staging (<date>)"
        4. If any tests fail, post to #qa-results:
           "Nightly E2E: <N> of <total> tests failed — <count> Linear tickets created"
           Include a list of the failing test names.
        ```
      </PromptBlock>

      Note the playbook ID after saving — you'll reference it in the automation's prompt. You can find it in the URL when viewing the playbook (`app.devin.ai/.../playbooks/{playbook_id}`).

      <Tip>Install the [Linear integration](/integrations/linear) so Devin can create tickets as part of the playbook. When editing the automation on the [Automations page](https://app.devin.ai/automations?utm_source=docs\&utm_medium=use-case-gallery), you can also add a **Post to Slack** notification (e.g., `#qa-results`) so your team gets notified automatically. Give Devin read-only access to your staging environment's secrets (database URLs, API keys) via [organization secrets](/product-guides/secrets) if your tests need them.</Tip>
    </Step>

    <Step title="Create the nightly automation via the API">
      Now use the `POST /v3/organizations/{org_id}/automations` endpoint to register an automation with a `schedule:recurring` trigger and a `start_session` action. Reference the playbook in the prompt with an `@playbook:{id}` token. Sessions started by an automation only get the tools you grant it, so the `tools` block enables Linear tools and lets Devin post to your `#qa-results` channel (replace the Slack workspace and channel IDs with your own). This example runs every night at 2 AM UTC:

      ```bash theme={null}
      curl -X POST "https://api.devin.ai/v3/organizations/$ORG_ID/automations" \
        -H "Authorization: Bearer $DEVIN_API_KEY" \
        -H "Content-Type: application/json" \
        -d '{
          "name": "Nightly E2E — staging",
          "run_as": {"type": "organization"},
          "triggers": [{
            "event_type": "schedule:recurring",
            "conditions": {"any": [{"all": [{
              "field": "rrule",
              "operator": "recurrence",
              "value": "FREQ=DAILY;BYHOUR=2;BYMINUTE=0"
            }]}]}
          }],
          "actions": [{
            "type": "start_session",
            "prompt": "Run the nightly E2E test suite against staging. Follow @playbook:your-playbook-id exactly."
          }],
          "tools": {
            "linear_enabled": true,
            "slack_channels": [{"team_id": "T0123456789", "channel_id": "C0123456789"}]
          }
        }'
      ```

      The response includes an `automation_id` you'll use to manage this automation later. Save it:

      ```bash theme={null}
      export AUTOMATION_ID="a1b2c3d4e5f6789012345678"  # automation_id from the response
      ```

      The `rrule` condition takes an iCalendar `RRULE` evaluated in UTC. Some useful alternatives:

      | Pattern | When it runs |
      | - | - |
      | `FREQ=DAILY;BYHOUR=2;BYMINUTE=0` | Every night at 2 AM UTC |
      | `FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=2;BYMINUTE=0` | Weeknights only (Mon-Fri) |
      | `FREQ=WEEKLY;BYDAY=MO;BYHOUR=6;BYMINUTE=0` | Every Monday at 6 AM UTC |

      Why 2 AM? You want tests to run after the last deploy of the day has settled on staging, but early enough that failures are visible when engineers start work. Adjust to match your team's timezone and deploy cadence.

      See the [Create automation endpoint docs](/api-reference/v3/automations/post-organizations-automations) for all available fields.
    </Step>

    <Step title="Verify the first run and tune the prompt">
      After the automation fires for the first time, check the session to make sure Devin ran the tests correctly and the output matches what you expect.

      1. Open the automation on the [Automations page](https://app.devin.ai/automations?utm_source=docs\&utm_medium=use-case-gallery) and follow the session link on its **Activity** tab
      2. Did the Playwright suite execute? Were Linear tickets created for real failures (not flaky tests)?
      3. Check the `#qa-results` Slack channel for the summary message

      Common issues on the first run and how to fix them:

      * **Devin can't access staging**: Add your staging environment variables (like `STAGING_API_KEY` or `DATABASE_URL`) as [organization secrets](/product-guides/secrets) so they're available in every session the automation starts
      * **Too many tickets from flaky tests**: Add a retry to your playbook: "Re-run any failing test once before filing a ticket. Only file tickets for tests that fail twice."
      * **Tests take too long**: Scope the suite — e.g., "Only run tests in `tests/critical/` and `tests/smoke/`" — or increase the session timeout

      <PromptBlock agent="advanced">
        ```txt Prompt tweak: filter out flaky tests theme={null}
        Update the nightly E2E playbook: before creating a Linear ticket for a failing
        test, re-run it once with `npx playwright test <test-file> --retries=1`.
        Only create a ticket if the test fails on retry. Add a "flaky" label to any
        test that passed on retry and mention it in the Slack summary.
        ```
      </PromptBlock>
    </Step>

    <Step title="Manage automations as code">
      Once your nightly run is stable, you'll want to manage it alongside your other automations — pausing during deploy freezes, updating the prompt when your test suite changes, or spinning up a second automation for a different environment.

      **Pause the automation** during a deploy freeze or maintenance window:

      ```bash theme={null}
      curl -X PATCH "https://api.devin.ai/v3/organizations/$ORG_ID/automations/$AUTOMATION_ID" \
        -H "Authorization: Bearer $DEVIN_API_KEY" \
        -H "Content-Type: application/json" \
        -d '{"enabled": false}'
      ```

      **Re-enable it** when the freeze ends:

      ```bash theme={null}
      curl -X PATCH "https://api.devin.ai/v3/organizations/$ORG_ID/automations/$AUTOMATION_ID" \
        -H "Authorization: Bearer $DEVIN_API_KEY" \
        -H "Content-Type: application/json" \
        -d '{"enabled": true}'
      ```

      **List all automations** to audit what's running:

      ```bash theme={null}
      curl "https://api.devin.ai/v3/organizations/$ORG_ID/automations" \
        -H "Authorization: Bearer $DEVIN_API_KEY"
      ```

      For teams managing multiple automations, ask Devin to build a CLI that syncs automation definitions from a YAML config file — so you can version-control your automations alongside your test configuration:

      <PromptBlock>
        ```txt Automations-as-code CLI theme={null}
        Write a Python script that reads scheduled automation definitions from a
        file called devin-automations.yaml and syncs them to the Devin API.

        The YAML file should look like:
          automations:
            - name: "Nightly E2E — staging"
              prompt: "Run the nightly E2E suite against staging. Follow @playbook:a1b2c3d4e5f6789012345678 exactly."
              rrule: "FREQ=DAILY;BYHOUR=2;BYMINUTE=0"
            - name: "Nightly E2E — preview"
              prompt: "Run the E2E suite against the preview environment. Follow @playbook:a1b2c3d4e5f6789012345678 exactly."
              rrule: "FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR;BYHOUR=3;BYMINUTE=0"

        The script should:
        1. Compare the YAML definitions against existing automations from the API
        2. Create new automations (with a schedule:recurring trigger, an rrule
           condition, a start_session action, and run_as organization), update
           changed ones, and disable automations removed from the file
        3. Support a --dry-run flag that shows what would change
        4. Use the endpoints at /v3/organizations/{org_id}/automations
        ```
      </PromptBlock>
    </Step>
  </Steps>
</div>


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.