crypto/byteclaw
1
0
Fork 0
Code Issues Pull requests Projects Releases 1 Packages Wiki Activity Actions

feat: config, OpenAI-compatible streaming provider and safe tools

Browse source
This commit is contained in:
OpenClaw Dev 2026-10-25 03:10:13 +01:00
commit c1c7b0e005
13 changed files with 1031 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored Normal file
View file

@ -0,0 +1,6 @@
data/
bin/
dist/
*.db
config.yaml
.env

5
go.mod Normal file
View file

@ -0,0 +1,5 @@
module github.com/crypto/openclaw
go 1.25
require gopkg.in/yaml.v3 v3.0.1 // indirect

3
go.sum Normal file
View file

@ -0,0 +1,3 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

143
internal/config/config.go Normal file
View file

@ -0,0 +1,143 @@
// Package config loads OpenClaw configuration from YAML/JSON files and env vars.
package config
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"gopkg.in/yaml.v3"
)
type LLM struct {
BaseURL string `yaml:"base_url" json:"base_url"`
APIKey string `yaml:"api_key" json:"api_key"`
Model string `yaml:"model" json:"model"`
TimeoutSec int `yaml:"timeout_sec" json:"timeout_sec"`
Temperature float64 `yaml:"temperature" json:"temperature"`
}
type Agent struct {
SystemPrompt string `yaml:"system_prompt" json:"system_prompt"`
MaxIterations int `yaml:"max_iterations" json:"max_iterations"`
HistoryLimit int `yaml:"history_limit" json:"history_limit"`
}
type WebFetch struct {
Enabled bool `yaml:"enabled" json:"enabled"`
// MaxBytes caps the response body read from remote servers.
MaxBytes int64 `yaml:"max_bytes" json:"max_bytes"`
TimeoutSec int `yaml:"timeout_sec" json:"timeout_sec"`
// AllowPrivate permits fetching loopback/private/link-local addresses (SSRF risk). Default false.
AllowPrivate bool `yaml:"allow_private" json:"allow_private"`
}
type Tools struct {
WebFetch WebFetch `yaml:"web_fetch" json:"web_fetch"`
// FilesEnabled enables read/write/list tools confined to the workspace directory.
FilesEnabled bool `yaml:"files_enabled" json:"files_enabled"`
// FileMaxBytes caps file reads/writes.
FileMaxBytes int64 `yaml:"file_max_bytes" json:"file_max_bytes"`
}
type Config struct {
Listen string `yaml:"listen" json:"listen"`
AuthToken string `yaml:"auth_token" json:"auth_token"`
// AllowedOrigins for browser WebSocket/CORS. Empty = same host only. "*" must be set explicitly.
AllowedOrigins []string `yaml:"allowed_origins" json:"allowed_origins"`
DataDir string `yaml:"data_dir" json:"data_dir"`
Workspace string `yaml:"workspace" json:"workspace"`
LLM LLM `yaml:"llm" json:"llm"`
Agent Agent `yaml:"agent" json:"agent"`
Tools Tools `yaml:"tools" json:"tools"`
}
// Default returns a config with secure defaults.
func Default() Config {
return Config{
Listen: "127.0.0.1:8787",
DataDir: "./data",
LLM: LLM{
BaseURL: "https://api.openai.com/v1",
Model: "gpt-4o-mini",
TimeoutSec: 120,
Temperature: 0.7,
},
Agent: Agent{
SystemPrompt: "You are OpenClaw, a concise and helpful self-hosted assistant. Use tools when they help.",
MaxIterations: 8,
HistoryLimit: 50,
},
Tools: Tools{
WebFetch: WebFetch{Enabled: true, MaxBytes: 512 * 1024, TimeoutSec: 15},
FilesEnabled: true,
FileMaxBytes: 1 << 20,
},
}
}
// Load reads path (optional; .json or .yaml/.yml) on top of defaults, then applies env overrides.
func Load(path string) (Config, error) {
cfg := Default()
if path != "" {
b, err := os.ReadFile(path)
if err != nil {
return cfg, fmt.Errorf("read config: %w", err)
}
switch strings.ToLower(filepath.Ext(path)) {
case ".json":
err = json.Unmarshal(b, &cfg)
default:
err = yaml.Unmarshal(b, &cfg)
}
if err != nil {
return cfg, fmt.Errorf("parse config: %w", err)
}
}
applyEnv(&cfg, os.Getenv)
if cfg.Workspace == "" {
cfg.Workspace = filepath.Join(cfg.DataDir, "workspace")
}
return cfg, cfg.Validate()
}
func applyEnv(c *Config, get func(string) string) {
set := func(key string, dst *string) {
if v := get(key); v != "" {
*dst = v
}
}
set("OPENCLAW_LISTEN", &c.Listen)
set("OPENCLAW_AUTH_TOKEN", &c.AuthToken)
set("OPENCLAW_DATA_DIR", &c.DataDir)
set("OPENCLAW_WORKSPACE", &c.Workspace)
set("OPENCLAW_LLM_BASE_URL", &c.LLM.BaseURL)
set("OPENCLAW_LLM_API_KEY", &c.LLM.APIKey)
set("OPENCLAW_LLM_MODEL", &c.LLM.Model)
if v := get("OPENCLAW_ALLOWED_ORIGINS"); v != "" {
c.AllowedOrigins = nil
for _, o := range strings.Split(v, ",") {
if o = strings.TrimSpace(o); o != "" {
c.AllowedOrigins = append(c.AllowedOrigins, o)
}
}
}
}
// Validate enforces required fields and secure settings.
func (c Config) Validate() error {
var errs []error
if len(c.AuthToken) < 16 {
errs = append(errs, errors.New("auth_token must be set and at least 16 characters (OPENCLAW_AUTH_TOKEN)"))
}
if c.LLM.BaseURL == "" || c.LLM.Model == "" {
errs = append(errs, errors.New("llm.base_url and llm.model are required"))
}
if c.Agent.MaxIterations <= 0 {
errs = append(errs, errors.New("agent.max_iterations must be > 0"))
}
return errors.Join(errs...)
}

44
internal/config/config_test.go Normal file
View file

@ -0,0 +1,44 @@
package config
import (
"os"
"path/filepath"
"testing"
)
func TestLoadYAMLAndEnv(t *testing.T) {
dir := t.TempDir()
p := filepath.Join(dir, "c.yaml")
os.WriteFile(p, []byte("auth_token: aaaaaaaaaaaaaaaaaaaa\nllm:\n model: llama3\n"), 0o600)
t.Setenv("OPENCLAW_LLM_BASE_URL", "http://localhost:11434/v1")
t.Setenv("OPENCLAW_ALLOWED_ORIGINS", "https://a.example, https://b.example")
c, err := Load(p)
if err != nil {
t.Fatal(err)
}
if c.LLM.Model != "llama3" || c.LLM.BaseURL != "http://localhost:11434/v1" {
t.Fatalf("unexpected llm cfg %+v", c.LLM)
}
if len(c.AllowedOrigins) != 2 || c.AllowedOrigins[1] != "https://b.example" {
t.Fatalf("origins %v", c.AllowedOrigins)
}
if c.Listen != "127.0.0.1:8787" {
t.Fatalf("default listen changed: %s", c.Listen)
}
}
func TestLoadJSON(t *testing.T) {
p := filepath.Join(t.TempDir(), "c.json")
os.WriteFile(p, []byte(`{"auth_token":"bbbbbbbbbbbbbbbbbbbb","listen":":9000"}`), 0o600)
c, err := Load(p)
if err != nil || c.Listen != ":9000" {
t.Fatalf("%v %v", c.Listen, err)
}
}
func TestValidateRequiresToken(t *testing.T) {
t.Setenv("OPENCLAW_AUTH_TOKEN", "")
if _, err := Load(""); err == nil {
t.Fatal("expected error without token")
}
}

53
internal/llm/llm.go Normal file
View file

@ -0,0 +1,53 @@
// Package llm defines the provider abstraction and message types.
package llm
import "context"
type Role string
const (
RoleSystem Role = "system"
RoleUser Role = "user"
RoleAssistant Role = "assistant"
RoleTool Role = "tool"
)
type ToolCall struct {
ID string `json:"id"`
Name string `json:"name"`
Arguments string `json:"arguments"` // raw JSON
}
type Message struct {
Role Role `json:"role"`
Content string `json:"content"`
ToolCalls []ToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Name string `json:"name,omitempty"`
}
// ToolSpec describes a callable tool using JSON Schema parameters.
type ToolSpec struct {
Name string `json:"name"`
Description string `json:"description"`
Parameters map[string]any `json:"parameters"`
}
type Request struct {
Messages []Message
Tools []ToolSpec
}
// Response is the final assembled assistant turn.
type Response struct {
Content string
ToolCalls []ToolCall
FinishReason string
}
// DeltaFunc receives streamed text tokens. May be nil.
type DeltaFunc func(text string)
type Provider interface {
Chat(ctx context.Context, req Request, onDelta DeltaFunc) (Response, error)
}

205
internal/llm/openai.go Normal file
View file

@ -0,0 +1,205 @@
package llm
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"sort"
"strings"
"time"
)
// OpenAI is a provider for any OpenAI-compatible /chat/completions endpoint
// (OpenAI, OpenRouter, Ollama, LM Studio, vLLM, ...). It always streams.
type OpenAI struct {
BaseURL string
APIKey string
Model string
Temperature float64
HTTP *http.Client
}
func NewOpenAI(baseURL, apiKey, model string, temperature float64, timeout time.Duration) *OpenAI {
// Default transport: TLS certificate verification stays ON.
return &OpenAI{
BaseURL: strings.TrimRight(baseURL, "/"), APIKey: apiKey, Model: model,
Temperature: temperature, HTTP: &http.Client{Timeout: timeout},
}
}
type oaTool struct {
Type string `json:"type"`
Function ToolSpec `json:"function"`
}
type oaFunc struct {
Name string `json:"name,omitempty"`
Arguments string `json:"arguments,omitempty"`
}
type oaToolCall struct {
Index *int `json:"index,omitempty"`
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
Function oaFunc `json:"function"`
}
type oaMsg struct {
Role string `json:"role"`
Content *string `json:"content"`
ToolCalls []oaToolCall `json:"tool_calls,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Name string `json:"name,omitempty"`
}
type oaReq struct {
Model string `json:"model"`
Messages []oaMsg `json:"messages"`
Tools []oaTool `json:"tools,omitempty"`
Stream bool `json:"stream"`
Temperature float64 `json:"temperature"`
}
type oaChunk struct {
Choices []struct {
Delta struct {
Content string `json:"content"`
ToolCalls []oaToolCall `json:"tool_calls"`
} `json:"delta"`
FinishReason *string `json:"finish_reason"`
} `json:"choices"`
Error *struct {
Message string `json:"message"`
} `json:"error"`
}
func toOA(msgs []Message) []oaMsg {
out := make([]oaMsg, 0, len(msgs))
for _, m := range msgs {
c := m.Content
om := oaMsg{Role: string(m.Role), Content: &c, ToolCallID: m.ToolCallID, Name: m.Name}
if m.Role == RoleAssistant && len(m.ToolCalls) > 0 && c == "" {
om.Content = nil
}
for _, tc := range m.ToolCalls {
om.ToolCalls = append(om.ToolCalls, oaToolCall{ID: tc.ID, Type: "function", Function: oaFunc{Name: tc.Name, Arguments: tc.Arguments}})
}
out = append(out, om)
}
return out
}
func (p *OpenAI) Chat(ctx context.Context, req Request, onDelta DeltaFunc) (Response, error) {
body := oaReq{Model: p.Model, Messages: toOA(req.Messages), Stream: true, Temperature: p.Temperature}
for _, t := range req.Tools {
body.Tools = append(body.Tools, oaTool{Type: "function", Function: t})
}
b, err := json.Marshal(body)
if err != nil {
return Response{}, err
}
hr, err := http.NewRequestWithContext(ctx, http.MethodPost, p.BaseURL+"/chat/completions", bytes.NewReader(b))
if err != nil {
return Response{}, err
}
hr.Header.Set("Content-Type", "application/json")
hr.Header.Set("Accept", "text/event-stream")
if p.APIKey != "" {
hr.Header.Set("Authorization", "Bearer "+p.APIKey)
}
resp, err := p.HTTP.Do(hr)
if err != nil {
return Response{}, fmt.Errorf("llm request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
return Response{}, fmt.Errorf("llm http %d: %s", resp.StatusCode, strings.TrimSpace(string(msg)))
}
return parseStream(resp.Body, onDelta)
}
type partialCall struct {
id, name string
args strings.Builder
}
func parseStream(r io.Reader, onDelta DeltaFunc) (Response, error) {
var out Response
var text strings.Builder
calls := map[int]*partialCall{}
sc := bufio.NewScanner(r)
sc.Buffer(make([]byte, 64*1024), 4<<20)
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
if !strings.HasPrefix(line, "data:") {
continue
}
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
if data == "[DONE]" {
break
}
var ch oaChunk
if err := json.Unmarshal([]byte(data), &ch); err != nil {
return out, fmt.Errorf("decode stream chunk: %w", err)
}
if ch.Error != nil {
return out, fmt.Errorf("llm error: %s", ch.Error.Message)
}
for _, c := range ch.Choices {
if c.Delta.Content != "" {
text.WriteString(c.Delta.Content)
if onDelta != nil {
onDelta(c.Delta.Content)
}
}
for i, tc := range c.Delta.ToolCalls {
idx := i
if tc.Index != nil {
idx = *tc.Index
}
pc := calls[idx]
if pc == nil {
pc = &partialCall{}
calls[idx] = pc
}
if tc.ID != "" {
pc.id = tc.ID
}
if tc.Function.Name != "" {
pc.name += tc.Function.Name
}
pc.args.WriteString(tc.Function.Arguments)
}
if c.FinishReason != nil {
out.FinishReason = *c.FinishReason
}
}
}
if err := sc.Err(); err != nil {
return out, fmt.Errorf("read stream: %w", err)
}
out.Content = text.String()
idxs := make([]int, 0, len(calls))
for i := range calls {
idxs = append(idxs, i)
}
sort.Ints(idxs)
for _, i := range idxs {
pc := calls[i]
id := pc.id
if id == "" {
id = fmt.Sprintf("call_%d", i)
}
args := pc.args.String()
if args == "" {
args = "{}"
}
out.ToolCalls = append(out.ToolCalls, ToolCall{ID: id, Name: pc.name, Arguments: args})
}
return out, nil
}

67
internal/llm/openai_test.go Normal file
View file

@ -0,0 +1,67 @@
package llm
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
)
func TestOpenAIStreamTextAndToolCalls(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer k" || r.URL.Path != "/v1/chat/completions" {
http.Error(w, "bad", 400)
return
}
var body map[string]any
json.NewDecoder(r.Body).Decode(&body)
if body["stream"] != true || len(body["tools"].([]any)) != 1 {
http.Error(w, "bad body", 400)
return
}
chunks := []string{
`{"choices":[{"delta":{"content":"Hel"}}]}`,
`{"choices":[{"delta":{"content":"lo"}}]}`,
`{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","function":{"name":"time_now","arguments":"{\"tz\":"}}]}}]}`,
`{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"UTC\"}"}}]},"finish_reason":"tool_calls"}]}`,
}
for _, c := range chunks {
fmt.Fprintf(w, "data: %s\n\n", c)
}
fmt.Fprint(w, "data: [DONE]\n\n")
}))
defer srv.Close()
p := NewOpenAI(srv.URL+"/v1/", "k", "m", 0, 5*time.Second)
var streamed strings.Builder
resp, err := p.Chat(context.Background(), Request{
Messages: []Message{{Role: RoleUser, Content: "hi"}},
Tools: []ToolSpec{{Name: "time_now", Parameters: map[string]any{"type": "object"}}},
}, func(s string) { streamed.WriteString(s) })
if err != nil {
t.Fatal(err)
}
if resp.Content != "Hello" || streamed.String() != "Hello" {
t.Fatalf("content %q streamed %q", resp.Content, streamed.String())
}
if len(resp.ToolCalls) != 1 || resp.ToolCalls[0].Arguments != `{"tz":"UTC"}` || resp.ToolCalls[0].ID != "c1" {
t.Fatalf("tool calls %+v", resp.ToolCalls)
}
if resp.FinishReason != "tool_calls" {
t.Fatal(resp.FinishReason)
}
}
func TestOpenAIHTTPError(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Error(w, "nope", 401)
}))
defer srv.Close()
_, err := NewOpenAI(srv.URL, "", "m", 0, time.Second).Chat(context.Background(), Request{}, nil)
if err == nil || !strings.Contains(err.Error(), "401") {
t.Fatalf("want 401 err, got %v", err)
}
}

183
internal/tools/files.go Normal file
View file

@ -0,0 +1,183 @@
package tools
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/crypto/openclaw/internal/llm"
)
// Workspace confines file access to a root directory using os.Root
// (Go 1.24+), which rejects "..", absolute paths and symlink escapes.
type Workspace struct {
Dir string
MaxBytes int64
}
func (w Workspace) open() (*os.Root, error) {
if err := os.MkdirAll(w.Dir, 0o750); err != nil {
return nil, err
}
return os.OpenRoot(w.Dir)
}
func cleanRel(p string) (string, error) {
if p == "" {
p = "."
}
if filepath.IsAbs(p) {
return "", errors.New("path must be relative to the workspace")
}
return filepath.Clean(p), nil
}
type pathArgs struct {
Path string `json:"path"`
Content string `json:"content"`
}
// ---- file_read
type FileRead struct{ Workspace }
func (FileRead) Spec() llm.ToolSpec {
return llm.ToolSpec{Name: "file_read", Description: "Read a text file inside the workspace.",
Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}}, "path")}
}
func (f FileRead) Call(_ context.Context, raw json.RawMessage) (string, error) {
var a pathArgs
if err := json.Unmarshal(raw, &a); err != nil {
return "", err
}
p, err := cleanRel(a.Path)
if err != nil {
return "", err
}
root, err := f.open()
if err != nil {
return "", err
}
defer root.Close()
fh, err := root.Open(p)
if err != nil {
return "", err
}
defer fh.Close()
b, err := io.ReadAll(io.LimitReader(fh, f.MaxBytes+1))
if err != nil {
return "", err
}
if int64(len(b)) > f.MaxBytes {
return string(b[:f.MaxBytes]) + "\n[truncated]", nil
}
return string(b), nil
}
// ---- file_write
type FileWrite struct{ Workspace }
func (FileWrite) Spec() llm.ToolSpec {
return llm.ToolSpec{Name: "file_write", Description: "Create or overwrite a text file inside the workspace.",
Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}, "content": map[string]any{"type": "string"}}, "path", "content")}
}
func (f FileWrite) Call(_ context.Context, raw json.RawMessage) (string, error) {
var a pathArgs
if err := json.Unmarshal(raw, &a); err != nil {
return "", err
}
if int64(len(a.Content)) > f.MaxBytes {
return "", fmt.Errorf("content exceeds %d bytes", f.MaxBytes)
}
p, err := cleanRel(a.Path)
if err != nil {
return "", err
}
if p == "." {
return "", errors.New("path required")
}
root, err := f.open()
if err != nil {
return "", err
}
defer root.Close()
if dir := filepath.Dir(p); dir != "." {
if err := mkdirAll(root, dir); err != nil {
return "", err
}
}
fh, err := root.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
if err != nil {
return "", err
}
defer fh.Close()
if _, err := fh.WriteString(a.Content); err != nil {
return "", err
}
return fmt.Sprintf("wrote %d bytes to %s", len(a.Content), p), nil
}
func mkdirAll(root *os.Root, dir string) error {
cur := ""
for _, part := range strings.Split(filepath.ToSlash(dir), "/") {
cur = filepath.Join(cur, part)
if err := root.Mkdir(cur, 0o750); err != nil && !errors.Is(err, os.ErrExist) {
return err
}
}
return nil
}
// ---- file_list
type FileList struct{ Workspace }
func (FileList) Spec() llm.ToolSpec {
return llm.ToolSpec{Name: "file_list", Description: "List entries of a directory inside the workspace (default: root).",
Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}})}
}
func (f FileList) Call(_ context.Context, raw json.RawMessage) (string, error) {
var a pathArgs
if err := json.Unmarshal(raw, &a); err != nil {
return "", err
}
p, err := cleanRel(a.Path)
if err != nil {
return "", err
}
root, err := f.open()
if err != nil {
return "", err
}
defer root.Close()
d, err := root.Open(p)
if err != nil {
return "", err
}
defer d.Close()
ents, err := d.ReadDir(500)
if err != nil && !errors.Is(err, io.EOF) {
return "", err
}
var sb strings.Builder
for _, e := range ents {
name := e.Name()
if e.IsDir() {
name += "/"
}
sb.WriteString(name + "\n")
}
if sb.Len() == 0 {
return "(empty)", nil
}
return sb.String(), nil
}

41
internal/tools/timetool.go Normal file
View file

@ -0,0 +1,41 @@
package tools
import (
"context"
"encoding/json"
"fmt"
"time"
"github.com/crypto/openclaw/internal/llm"
)
// TimeNow returns the current time, optionally in an IANA time zone.
type TimeNow struct{ Now func() time.Time }
func (TimeNow) Spec() llm.ToolSpec {
return llm.ToolSpec{
Name: "time_now",
Description: "Get the current date and time. Optional IANA time zone, e.g. Europe/Berlin.",
Parameters: obj(map[string]any{"timezone": map[string]any{"type": "string"}}),
}
}
func (t TimeNow) Call(_ context.Context, raw json.RawMessage) (string, error) {
var a struct{ Timezone string `json:"timezone"` }
if err := json.Unmarshal(raw, &a); err != nil {
return "", err
}
now := time.Now
if t.Now != nil {
now = t.Now
}
n := now()
if a.Timezone != "" {
loc, err := time.LoadLocation(a.Timezone)
if err != nil {
return "", fmt.Errorf("unknown timezone %q", a.Timezone)
}
n = n.In(loc)
}
return n.Format("2006-01-02T15:04:05 MST (Monday)"), nil
}

64
internal/tools/tools.go Normal file
View file

@ -0,0 +1,64 @@
// Package tools contains the tool registry and built-in safe tools.
package tools
import (
"context"
"encoding/json"
"fmt"
"sort"
"github.com/crypto/openclaw/internal/llm"
)
type Tool interface {
Spec() llm.ToolSpec
Call(ctx context.Context, args json.RawMessage) (string, error)
}
type Registry struct{ tools map[string]Tool }
func NewRegistry(ts ...Tool) *Registry {
r := &Registry{tools: map[string]Tool{}}
for _, t := range ts {
r.Register(t)
}
return r
}
func (r *Registry) Register(t Tool) { r.tools[t.Spec().Name] = t }
func (r *Registry) Specs() []llm.ToolSpec {
out := make([]llm.ToolSpec, 0, len(r.tools))
for _, t := range r.tools {
out = append(out, t.Spec())
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
// Call runs a tool; errors are returned as text so the model can react to them.
func (r *Registry) Call(ctx context.Context, name, args string) string {
t, ok := r.tools[name]
if !ok {
return fmt.Sprintf("error: unknown tool %q", name)
}
if args == "" {
args = "{}"
}
if !json.Valid([]byte(args)) {
return "error: arguments are not valid JSON"
}
res, err := t.Call(ctx, json.RawMessage(args))
if err != nil {
return "error: " + err.Error()
}
return res
}
func obj(props map[string]any, required ...string) map[string]any {
m := map[string]any{"type": "object", "properties": props}
if len(required) > 0 {
m["required"] = required
}
return m
}

92
internal/tools/tools_test.go Normal file
View file

@ -0,0 +1,92 @@
package tools
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func TestTimeNow(t *testing.T) {
fixed := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
out, err := TimeNow{Now: func() time.Time { return fixed }}.Call(context.Background(), json.RawMessage(`{"timezone":"Europe/Berlin"}`))
if err != nil || !strings.HasPrefix(out, "2026-01-02T04:04:05") {
t.Fatalf("%q %v", out, err)
}
if _, err := (TimeNow{}).Call(context.Background(), json.RawMessage(`{"timezone":"Nope/Nope"}`)); err == nil {
t.Fatal("expected tz error")
}
}
func TestWorkspaceConfinement(t *testing.T) {
dir := t.TempDir()
ws := Workspace{Dir: filepath.Join(dir, "ws"), MaxBytes: 100}
ctx := context.Background()
w, r, l := FileWrite{ws}, FileRead{ws}, FileList{ws}
if _, err := w.Call(ctx, json.RawMessage(`{"path":"a/b.txt","content":"hi"}`)); err != nil {
t.Fatal(err)
}
if out, err := r.Call(ctx, json.RawMessage(`{"path":"a/b.txt"}`)); err != nil || out != "hi" {
t.Fatalf("%q %v", out, err)
}
if out, _ := l.Call(ctx, json.RawMessage(`{}`)); out != "a/\n" {
t.Fatalf("list %q", out)
}
os.WriteFile(filepath.Join(dir, "secret"), []byte("x"), 0o600)
os.Symlink(filepath.Join(dir, "secret"), filepath.Join(ws.Dir, "link"))
for _, p := range []string{"../secret", "/etc/passwd", "a/../../secret", "link"} {
if _, err := r.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q}`, p))); err == nil {
t.Errorf("read of %q should fail", p)
}
if _, err := w.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q,"content":"x"}`, p))); err == nil {
t.Errorf("write of %q should fail", p)
}
}
if _, err := w.Call(ctx, json.RawMessage(`{"path":"big","content":"`+strings.Repeat("x", 101)+`"}`)); err == nil {
t.Error("size limit not enforced")
}
}
func TestWebFetchBlocksPrivateByDefault(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "ok") }))
defer srv.Close()
_, err := NewWebFetch(2*time.Second, 1024, false).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`))
if err == nil || !errors.Is(err, ErrBlockedAddr) {
t.Fatalf("expected blocked, got %v", err)
}
if _, err := NewWebFetch(time.Second, 10, false).Call(context.Background(), json.RawMessage(`{"url":"file:///etc/passwd"}`)); err == nil {
t.Fatal("file scheme must be rejected")
}
}
func TestWebFetchStripsAndTruncates(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
fmt.Fprint(w, "<html><script>evil()</script><p>Hello <b>World</b></p>"+strings.Repeat("y", 500)+"</html>")
}))
defer srv.Close()
out, err := NewWebFetch(2*time.Second, 80, true).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`))
if err != nil {
t.Fatal(err)
}
if strings.Contains(out, "evil") || !strings.Contains(out, "Hello World") || !strings.HasSuffix(out, "[truncated]") {
t.Fatalf("unexpected: %q", out)
}
}
func TestRegistryUnknownAndBadJSON(t *testing.T) {
r := NewRegistry(TimeNow{})
if !strings.Contains(r.Call(context.Background(), "nope", "{}"), "unknown tool") {
t.Fatal()
}
if !strings.Contains(r.Call(context.Background(), "time_now", "{bad"), "not valid JSON") {
t.Fatal()
}
}

125
internal/tools/webfetch.go Normal file
View file

@ -0,0 +1,125 @@
package tools
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"regexp"
"strings"
"syscall"
"time"
"github.com/crypto/openclaw/internal/llm"
)
// WebFetch downloads a URL with timeout and size limit. By default it refuses
// loopback/private/link-local targets (SSRF protection), checked at dial time
// so DNS rebinding and redirects are covered too. TLS verification is always on.
type WebFetch struct {
client *http.Client
maxBytes int64
}
var ErrBlockedAddr = errors.New("destination address is not allowed")
func NewWebFetch(timeout time.Duration, maxBytes int64, allowPrivate bool) *WebFetch {
d := &net.Dialer{Timeout: 10 * time.Second}
if !allowPrivate {
d.Control = func(_, address string, _ syscall.RawConn) error {
host, _, err := net.SplitHostPort(address)
if err != nil {
return err
}
ip := net.ParseIP(host)
if ip == nil || blockedIP(ip) {
return ErrBlockedAddr
}
return nil
}
}
tr := &http.Transport{DialContext: d.DialContext, TLSHandshakeTimeout: 10 * time.Second, ForceAttemptHTTP2: true}
return &WebFetch{maxBytes: maxBytes, client: &http.Client{
Timeout: timeout, Transport: tr,
CheckRedirect: func(req *http.Request, via []*http.Request) error {
if len(via) >= 5 {
return errors.New("too many redirects")
}
if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
return errors.New("redirect to unsupported scheme")
}
return nil
},
}}
}
func blockedIP(ip net.IP) bool {
return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
ip.IsUnspecified() || ip.IsMulticast() || ip.IsInterfaceLocalMulticast() ||
ip.Equal(net.IPv4bcast) || (ip.To4() != nil && ip.To4()[0] == 100 && ip.To4()[1]&0xC0 == 64) // CGNAT 100.64/10
}
func (w *WebFetch) Spec() llm.ToolSpec {
return llm.ToolSpec{
Name: "web_fetch",
Description: "Fetch a public http(s) URL and return its text content (HTML is stripped, output truncated).",
Parameters: obj(map[string]any{"url": map[string]any{"type": "string"}}, "url"),
}
}
var (
reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>`)
reTag = regexp.MustCompile(`(?s)<[^>]+>`)
reSpace = regexp.MustCompile(`[ \t\r\f\v]+`)
reNL = regexp.MustCompile(`\n\s*\n+`)
)
func (w *WebFetch) Call(ctx context.Context, raw json.RawMessage) (string, error) {
var a struct{ URL string `json:"url"` }
if err := json.Unmarshal(raw, &a); err != nil {
return "", err
}
u, err := url.Parse(a.URL)
if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
return "", fmt.Errorf("only absolute http(s) URLs are allowed")
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
return "", err
}
req.Header.Set("User-Agent", "OpenClaw/0.1 (+self-hosted assistant)")
resp, err := w.client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, w.maxBytes+1))
if err != nil {
return "", err
}
truncated := int64(len(body)) > w.maxBytes
if truncated {
body = body[:w.maxBytes]
}
text := string(body)
if strings.Contains(resp.Header.Get("Content-Type"), "html") {
text = reScript.ReplaceAllString(text, "")
text = reTag.ReplaceAllString(text, " ")
text = reSpace.ReplaceAllString(text, " ")
text = reNL.ReplaceAllString(text, "\n\n")
}
text = strings.TrimSpace(text)
const maxOut = 20000
if len(text) > maxOut {
text, truncated = text[:maxOut], true
}
out := fmt.Sprintf("HTTP %d %s\n\n%s", resp.StatusCode, resp.Header.Get("Content-Type"), text)
if truncated {
out += "\n\n[truncated]"
}
return out, nil
}