Loading...
Loading...
Build AI agents that interact with computers like humans do - viewing screens, moving cursors, clicking buttons, and typing text. Covers Anthropic's Computer Use, OpenAI's Operator/CUA, and open-source alternatives. Critical focus on sandboxing, security, and handling the unique challenges of vision-based control. Use when: computer use, desktop automation agent, screen control AI, vision-based agent, GUI automation.
npx skill4agent add sickn33/antigravity-awesome-skills computer-use-agentsfrom anthropic import Anthropic
from PIL import Image
import base64
import pyautogui
import time
class ComputerUseAgent:
"""
Perception-Reasoning-Action loop implementation.
Based on Anthropic Computer Use patterns.
"""
def __init__(self, client: Anthropic, model: str = "claude-sonnet-4-20250514"):
self.client = client
self.model = model
self.max_steps = 50 # Prevent runaway loops
self.action_delay = 0.5 # Seconds between actions
def capture_screenshot(self) -> str:
"""Capture screen and return base64 encoded image."""
screenshot = pyautogui.screenshot()
# Resize for token efficiency (1280x800 is good balance)
screenshot = screenshot.resize((1280, 800), Image.LANCZOS)
import io
buffer = io.BytesIO()
screenshot.save(buffer, format="PNG")
return base64.b64encode(buffer.getvalue()).decode()
def execute_action(self, action: dict) -> dict:
"""Execute mouse/keyboard action on the computer."""
action_type = action.get("type")
if action_type == "click":
x, y = action["x"], action["y"]
button = action.get("button", "left")
pyautogui.click(x, y, button=button)
return {"success": True, "action": f"clicked at ({x}, {y})"}
elif action_type == "type":
text = action["text"]
pyautogui.typewrite(text, interval=0.02)
return {"success": True, "action": f"typed {len(text)} chars"}
elif action_type == "key":
key = action["key"]
pyautogui.press(key)
return {"success": True, "action": f"pressed {key}"}
elif action_type == "scroll":
direction = action.get("direction", "down")
amount = action.get("amount", 3)
scroll = -amount if direction == "down" else amount
pyautogui.scroll(scroll)
return {"success": True, "action": f"scrolled {dir# Dockerfile for sandboxed computer use environment
# Based on Anthropic's reference implementation pattern
FROM ubuntu:22.04
# Install desktop environment
RUN apt-get update && apt-get install -y \
xvfb \
x11vnc \
fluxbox \
xterm \
firefox \
python3 \
python3-pip \
supervisor
# Security: Create non-root user
RUN useradd -m -s /bin/bash agent && \
mkdir -p /home/agent/.vnc
# Install Python dependencies
COPY requirements.txt /tmp/
RUN pip3 install -r /tmp/requirements.txt
# Security: Drop capabilities
RUN apt-get install -y --no-install-recommends libcap2-bin && \
setcap -r /usr/bin/python3 || true
# Copy agent code
COPY --chown=agent:agent . /app
WORKDIR /app
# Supervisor config for virtual display + VNC
COPY supervisord.conf /etc/supervisor/conf.d/
# Expose VNC port only (not desktop directly)
EXPOSE 5900
# Run as non-root
USER agent
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
---
# docker-compose.yml with security constraints
version: '3.8'
services:
computer-use-agent:
build: .
ports:
- "5900:5900" # VNC for observation
- "8080:8080" # API for control
# Security constraints
security_opt:
- no-new-privileges:true
- seccomp:seccomp-profile.json
# Resource limits
deploy:
resources:
limits:
cpus: '2'
memory: 4G
reservations:
cpus: '0.5'
memory: 1G
# Network isolation
networks:
- agent-network
# No access to host filesystem
volumes:
- agent-tmp:/tmp
# Read-only root filesystem
read_only: true
tmpfs:
- /run
- /var/run
# Environment
environment:
- DISPLAY=:99
- NO_PROXY=localhost
networks:
agent-network:
driver: bridge
internal: true # No internet by default
volumes:
agent-tmp:
---
# Python wrapper with additional runtime sandboxing
import subprocess
import os
from dataclasses imfrom anthropic import Anthropic
from anthropic.types.beta import (
BetaToolComputerUse20241022,
BetaToolBash20241022,
BetaToolTextEditor20241022,
)
import subprocess
import base64
from PIL import Image
import io
class AnthropicComputerUse:
"""
Official Anthropic Computer Use implementation.
Requires:
- Docker container with virtual display
- VNC for viewing agent actions
- Proper tool implementations
"""
def __init__(self):
self.client = Anthropic()
self.model = "claude-sonnet-4-20250514" # Best for computer use
self.screen_size = (1280, 800)
def get_tools(self) -> list:
"""Define computer use tools."""
return [
BetaToolComputerUse20241022(
type="computer_20241022",
name="computer",
display_width_px=self.screen_size[0],
display_height_px=self.screen_size[1],
),
BetaToolBash20241022(
type="bash_20241022",
name="bash",
),
BetaToolTextEditor20241022(
type="text_editor_20241022",
name="str_replace_editor",
),
]
def execute_tool(self, name: str, input: dict) -> dict:
"""Execute a tool and return result."""
if name == "computer":
return self._handle_computer_action(input)
elif name == "bash":
return self._handle_bash(input)
elif name == "str_replace_editor":
return self._handle_editor(input)
else:
return {"error": f"Unknown tool: {name}"}
def _handle_computer_action(self, input: dict) -> dict:
"""Handle computer control actions."""
action = input.get("action")
if action == "screenshot":
# Capture via xdotool/scrot
subprocess.run(["scrot", "/tmp/screenshot.png"])
with open("/tmp/screenshot.png", "rb") as f:
| Issue | Severity | Solution |
|---|---|---|
| Issue | critical | ## Defense in depth - no single solution works |
| Issue | medium | ## Add human-like variance to actions |
| Issue | high | ## Use keyboard alternatives when possible |
| Issue | medium | ## Accept the tradeoff |
| Issue | high | ## Implement context management |
| Issue | high | ## Monitor and limit costs |
| Issue | critical | ## ALWAYS use sandboxing |