From c1c7b0e005fdc706c0354e9f744e792257f91008 Mon Sep 17 00:00:00 2001 From: OpenClaw Dev Date: Sun, 25 Oct 2026 03:10:13 +0100 Subject: [PATCH] feat: config, OpenAI-compatible streaming provider and safe tools --- .gitignore | 6 + go.mod | 5 + go.sum | 3 + internal/config/config.go | 143 +++++++++++++++++++++++ internal/config/config_test.go | 44 +++++++ internal/llm/llm.go | 53 +++++++++ internal/llm/openai.go | 205 +++++++++++++++++++++++++++++++++ internal/llm/openai_test.go | 67 +++++++++++ internal/tools/files.go | 183 +++++++++++++++++++++++++++++ internal/tools/timetool.go | 41 +++++++ internal/tools/tools.go | 64 ++++++++++ internal/tools/tools_test.go | 92 +++++++++++++++ internal/tools/webfetch.go | 125 ++++++++++++++++++++ 13 files changed, 1031 insertions(+) create mode 100644 .gitignore create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/llm/llm.go create mode 100644 internal/llm/openai.go create mode 100644 internal/llm/openai_test.go create mode 100644 internal/tools/files.go create mode 100644 internal/tools/timetool.go create mode 100644 internal/tools/tools.go create mode 100644 internal/tools/tools_test.go create mode 100644 internal/tools/webfetch.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5131913 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +data/ +bin/ +dist/ +*.db +config.yaml +.env diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e97e379 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/crypto/openclaw + +go 1.25 + +require gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4bc0337 --- /dev/null +++ b/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..6df532f --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,143 @@ +// Package config loads OpenClaw configuration from YAML/JSON files and env vars. +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +type LLM struct { + BaseURL string `yaml:"base_url" json:"base_url"` + APIKey string `yaml:"api_key" json:"api_key"` + Model string `yaml:"model" json:"model"` + TimeoutSec int `yaml:"timeout_sec" json:"timeout_sec"` + Temperature float64 `yaml:"temperature" json:"temperature"` +} + +type Agent struct { + SystemPrompt string `yaml:"system_prompt" json:"system_prompt"` + MaxIterations int `yaml:"max_iterations" json:"max_iterations"` + HistoryLimit int `yaml:"history_limit" json:"history_limit"` +} + +type WebFetch struct { + Enabled bool `yaml:"enabled" json:"enabled"` + // MaxBytes caps the response body read from remote servers. + MaxBytes int64 `yaml:"max_bytes" json:"max_bytes"` + TimeoutSec int `yaml:"timeout_sec" json:"timeout_sec"` + // AllowPrivate permits fetching loopback/private/link-local addresses (SSRF risk). Default false. + AllowPrivate bool `yaml:"allow_private" json:"allow_private"` +} + +type Tools struct { + WebFetch WebFetch `yaml:"web_fetch" json:"web_fetch"` + // FilesEnabled enables read/write/list tools confined to the workspace directory. + FilesEnabled bool `yaml:"files_enabled" json:"files_enabled"` + // FileMaxBytes caps file reads/writes. + FileMaxBytes int64 `yaml:"file_max_bytes" json:"file_max_bytes"` +} + +type Config struct { + Listen string `yaml:"listen" json:"listen"` + AuthToken string `yaml:"auth_token" json:"auth_token"` + // AllowedOrigins for browser WebSocket/CORS. Empty = same host only. "*" must be set explicitly. + AllowedOrigins []string `yaml:"allowed_origins" json:"allowed_origins"` + DataDir string `yaml:"data_dir" json:"data_dir"` + Workspace string `yaml:"workspace" json:"workspace"` + LLM LLM `yaml:"llm" json:"llm"` + Agent Agent `yaml:"agent" json:"agent"` + Tools Tools `yaml:"tools" json:"tools"` +} + +// Default returns a config with secure defaults. +func Default() Config { + return Config{ + Listen: "127.0.0.1:8787", + DataDir: "./data", + LLM: LLM{ + BaseURL: "https://api.openai.com/v1", + Model: "gpt-4o-mini", + TimeoutSec: 120, + Temperature: 0.7, + }, + Agent: Agent{ + SystemPrompt: "You are OpenClaw, a concise and helpful self-hosted assistant. Use tools when they help.", + MaxIterations: 8, + HistoryLimit: 50, + }, + Tools: Tools{ + WebFetch: WebFetch{Enabled: true, MaxBytes: 512 * 1024, TimeoutSec: 15}, + FilesEnabled: true, + FileMaxBytes: 1 << 20, + }, + } +} + +// Load reads path (optional; .json or .yaml/.yml) on top of defaults, then applies env overrides. +func Load(path string) (Config, error) { + cfg := Default() + if path != "" { + b, err := os.ReadFile(path) + if err != nil { + return cfg, fmt.Errorf("read config: %w", err) + } + switch strings.ToLower(filepath.Ext(path)) { + case ".json": + err = json.Unmarshal(b, &cfg) + default: + err = yaml.Unmarshal(b, &cfg) + } + if err != nil { + return cfg, fmt.Errorf("parse config: %w", err) + } + } + applyEnv(&cfg, os.Getenv) + if cfg.Workspace == "" { + cfg.Workspace = filepath.Join(cfg.DataDir, "workspace") + } + return cfg, cfg.Validate() +} + +func applyEnv(c *Config, get func(string) string) { + set := func(key string, dst *string) { + if v := get(key); v != "" { + *dst = v + } + } + set("OPENCLAW_LISTEN", &c.Listen) + set("OPENCLAW_AUTH_TOKEN", &c.AuthToken) + set("OPENCLAW_DATA_DIR", &c.DataDir) + set("OPENCLAW_WORKSPACE", &c.Workspace) + set("OPENCLAW_LLM_BASE_URL", &c.LLM.BaseURL) + set("OPENCLAW_LLM_API_KEY", &c.LLM.APIKey) + set("OPENCLAW_LLM_MODEL", &c.LLM.Model) + if v := get("OPENCLAW_ALLOWED_ORIGINS"); v != "" { + c.AllowedOrigins = nil + for _, o := range strings.Split(v, ",") { + if o = strings.TrimSpace(o); o != "" { + c.AllowedOrigins = append(c.AllowedOrigins, o) + } + } + } +} + +// Validate enforces required fields and secure settings. +func (c Config) Validate() error { + var errs []error + if len(c.AuthToken) < 16 { + errs = append(errs, errors.New("auth_token must be set and at least 16 characters (OPENCLAW_AUTH_TOKEN)")) + } + if c.LLM.BaseURL == "" || c.LLM.Model == "" { + errs = append(errs, errors.New("llm.base_url and llm.model are required")) + } + if c.Agent.MaxIterations <= 0 { + errs = append(errs, errors.New("agent.max_iterations must be > 0")) + } + return errors.Join(errs...) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..99e3f3a --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,44 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +func TestLoadYAMLAndEnv(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "c.yaml") + os.WriteFile(p, []byte("auth_token: aaaaaaaaaaaaaaaaaaaa\nllm:\n model: llama3\n"), 0o600) + t.Setenv("OPENCLAW_LLM_BASE_URL", "http://localhost:11434/v1") + t.Setenv("OPENCLAW_ALLOWED_ORIGINS", "https://a.example, https://b.example") + c, err := Load(p) + if err != nil { + t.Fatal(err) + } + if c.LLM.Model != "llama3" || c.LLM.BaseURL != "http://localhost:11434/v1" { + t.Fatalf("unexpected llm cfg %+v", c.LLM) + } + if len(c.AllowedOrigins) != 2 || c.AllowedOrigins[1] != "https://b.example" { + t.Fatalf("origins %v", c.AllowedOrigins) + } + if c.Listen != "127.0.0.1:8787" { + t.Fatalf("default listen changed: %s", c.Listen) + } +} + +func TestLoadJSON(t *testing.T) { + p := filepath.Join(t.TempDir(), "c.json") + os.WriteFile(p, []byte(`{"auth_token":"bbbbbbbbbbbbbbbbbbbb","listen":":9000"}`), 0o600) + c, err := Load(p) + if err != nil || c.Listen != ":9000" { + t.Fatalf("%v %v", c.Listen, err) + } +} + +func TestValidateRequiresToken(t *testing.T) { + t.Setenv("OPENCLAW_AUTH_TOKEN", "") + if _, err := Load(""); err == nil { + t.Fatal("expected error without token") + } +} diff --git a/internal/llm/llm.go b/internal/llm/llm.go new file mode 100644 index 0000000..5c0772b --- /dev/null +++ b/internal/llm/llm.go @@ -0,0 +1,53 @@ +// Package llm defines the provider abstraction and message types. +package llm + +import "context" + +type Role string + +const ( + RoleSystem Role = "system" + RoleUser Role = "user" + RoleAssistant Role = "assistant" + RoleTool Role = "tool" +) + +type ToolCall struct { + ID string `json:"id"` + Name string `json:"name"` + Arguments string `json:"arguments"` // raw JSON +} + +type Message struct { + Role Role `json:"role"` + Content string `json:"content"` + ToolCalls []ToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` +} + +// ToolSpec describes a callable tool using JSON Schema parameters. +type ToolSpec struct { + Name string `json:"name"` + Description string `json:"description"` + Parameters map[string]any `json:"parameters"` +} + +type Request struct { + Messages []Message + Tools []ToolSpec +} + +// Response is the final assembled assistant turn. +type Response struct { + Content string + ToolCalls []ToolCall + FinishReason string +} + +// DeltaFunc receives streamed text tokens. May be nil. +type DeltaFunc func(text string) + +type Provider interface { + Chat(ctx context.Context, req Request, onDelta DeltaFunc) (Response, error) +} diff --git a/internal/llm/openai.go b/internal/llm/openai.go new file mode 100644 index 0000000..6c91ebf --- /dev/null +++ b/internal/llm/openai.go @@ -0,0 +1,205 @@ +package llm + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "sort" + "strings" + "time" +) + +// OpenAI is a provider for any OpenAI-compatible /chat/completions endpoint +// (OpenAI, OpenRouter, Ollama, LM Studio, vLLM, ...). It always streams. +type OpenAI struct { + BaseURL string + APIKey string + Model string + Temperature float64 + HTTP *http.Client +} + +func NewOpenAI(baseURL, apiKey, model string, temperature float64, timeout time.Duration) *OpenAI { + // Default transport: TLS certificate verification stays ON. + return &OpenAI{ + BaseURL: strings.TrimRight(baseURL, "/"), APIKey: apiKey, Model: model, + Temperature: temperature, HTTP: &http.Client{Timeout: timeout}, + } +} + +type oaTool struct { + Type string `json:"type"` + Function ToolSpec `json:"function"` +} + +type oaFunc struct { + Name string `json:"name,omitempty"` + Arguments string `json:"arguments,omitempty"` +} + +type oaToolCall struct { + Index *int `json:"index,omitempty"` + ID string `json:"id,omitempty"` + Type string `json:"type,omitempty"` + Function oaFunc `json:"function"` +} + +type oaMsg struct { + Role string `json:"role"` + Content *string `json:"content"` + ToolCalls []oaToolCall `json:"tool_calls,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + Name string `json:"name,omitempty"` +} + +type oaReq struct { + Model string `json:"model"` + Messages []oaMsg `json:"messages"` + Tools []oaTool `json:"tools,omitempty"` + Stream bool `json:"stream"` + Temperature float64 `json:"temperature"` +} + +type oaChunk struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + ToolCalls []oaToolCall `json:"tool_calls"` + } `json:"delta"` + FinishReason *string `json:"finish_reason"` + } `json:"choices"` + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +func toOA(msgs []Message) []oaMsg { + out := make([]oaMsg, 0, len(msgs)) + for _, m := range msgs { + c := m.Content + om := oaMsg{Role: string(m.Role), Content: &c, ToolCallID: m.ToolCallID, Name: m.Name} + if m.Role == RoleAssistant && len(m.ToolCalls) > 0 && c == "" { + om.Content = nil + } + for _, tc := range m.ToolCalls { + om.ToolCalls = append(om.ToolCalls, oaToolCall{ID: tc.ID, Type: "function", Function: oaFunc{Name: tc.Name, Arguments: tc.Arguments}}) + } + out = append(out, om) + } + return out +} + +func (p *OpenAI) Chat(ctx context.Context, req Request, onDelta DeltaFunc) (Response, error) { + body := oaReq{Model: p.Model, Messages: toOA(req.Messages), Stream: true, Temperature: p.Temperature} + for _, t := range req.Tools { + body.Tools = append(body.Tools, oaTool{Type: "function", Function: t}) + } + b, err := json.Marshal(body) + if err != nil { + return Response{}, err + } + hr, err := http.NewRequestWithContext(ctx, http.MethodPost, p.BaseURL+"/chat/completions", bytes.NewReader(b)) + if err != nil { + return Response{}, err + } + hr.Header.Set("Content-Type", "application/json") + hr.Header.Set("Accept", "text/event-stream") + if p.APIKey != "" { + hr.Header.Set("Authorization", "Bearer "+p.APIKey) + } + resp, err := p.HTTP.Do(hr) + if err != nil { + return Response{}, fmt.Errorf("llm request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return Response{}, fmt.Errorf("llm http %d: %s", resp.StatusCode, strings.TrimSpace(string(msg))) + } + return parseStream(resp.Body, onDelta) +} + +type partialCall struct { + id, name string + args strings.Builder +} + +func parseStream(r io.Reader, onDelta DeltaFunc) (Response, error) { + var out Response + var text strings.Builder + calls := map[int]*partialCall{} + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 4<<20) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if !strings.HasPrefix(line, "data:") { + continue + } + data := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if data == "[DONE]" { + break + } + var ch oaChunk + if err := json.Unmarshal([]byte(data), &ch); err != nil { + return out, fmt.Errorf("decode stream chunk: %w", err) + } + if ch.Error != nil { + return out, fmt.Errorf("llm error: %s", ch.Error.Message) + } + for _, c := range ch.Choices { + if c.Delta.Content != "" { + text.WriteString(c.Delta.Content) + if onDelta != nil { + onDelta(c.Delta.Content) + } + } + for i, tc := range c.Delta.ToolCalls { + idx := i + if tc.Index != nil { + idx = *tc.Index + } + pc := calls[idx] + if pc == nil { + pc = &partialCall{} + calls[idx] = pc + } + if tc.ID != "" { + pc.id = tc.ID + } + if tc.Function.Name != "" { + pc.name += tc.Function.Name + } + pc.args.WriteString(tc.Function.Arguments) + } + if c.FinishReason != nil { + out.FinishReason = *c.FinishReason + } + } + } + if err := sc.Err(); err != nil { + return out, fmt.Errorf("read stream: %w", err) + } + out.Content = text.String() + idxs := make([]int, 0, len(calls)) + for i := range calls { + idxs = append(idxs, i) + } + sort.Ints(idxs) + for _, i := range idxs { + pc := calls[i] + id := pc.id + if id == "" { + id = fmt.Sprintf("call_%d", i) + } + args := pc.args.String() + if args == "" { + args = "{}" + } + out.ToolCalls = append(out.ToolCalls, ToolCall{ID: id, Name: pc.name, Arguments: args}) + } + return out, nil +} diff --git a/internal/llm/openai_test.go b/internal/llm/openai_test.go new file mode 100644 index 0000000..9e972da --- /dev/null +++ b/internal/llm/openai_test.go @@ -0,0 +1,67 @@ +package llm + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestOpenAIStreamTextAndToolCalls(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer k" || r.URL.Path != "/v1/chat/completions" { + http.Error(w, "bad", 400) + return + } + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + if body["stream"] != true || len(body["tools"].([]any)) != 1 { + http.Error(w, "bad body", 400) + return + } + chunks := []string{ + `{"choices":[{"delta":{"content":"Hel"}}]}`, + `{"choices":[{"delta":{"content":"lo"}}]}`, + `{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","function":{"name":"time_now","arguments":"{\"tz\":"}}]}}]}`, + `{"choices":[{"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"UTC\"}"}}]},"finish_reason":"tool_calls"}]}`, + } + for _, c := range chunks { + fmt.Fprintf(w, "data: %s\n\n", c) + } + fmt.Fprint(w, "data: [DONE]\n\n") + })) + defer srv.Close() + p := NewOpenAI(srv.URL+"/v1/", "k", "m", 0, 5*time.Second) + var streamed strings.Builder + resp, err := p.Chat(context.Background(), Request{ + Messages: []Message{{Role: RoleUser, Content: "hi"}}, + Tools: []ToolSpec{{Name: "time_now", Parameters: map[string]any{"type": "object"}}}, + }, func(s string) { streamed.WriteString(s) }) + if err != nil { + t.Fatal(err) + } + if resp.Content != "Hello" || streamed.String() != "Hello" { + t.Fatalf("content %q streamed %q", resp.Content, streamed.String()) + } + if len(resp.ToolCalls) != 1 || resp.ToolCalls[0].Arguments != `{"tz":"UTC"}` || resp.ToolCalls[0].ID != "c1" { + t.Fatalf("tool calls %+v", resp.ToolCalls) + } + if resp.FinishReason != "tool_calls" { + t.Fatal(resp.FinishReason) + } +} + +func TestOpenAIHTTPError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "nope", 401) + })) + defer srv.Close() + _, err := NewOpenAI(srv.URL, "", "m", 0, time.Second).Chat(context.Background(), Request{}, nil) + if err == nil || !strings.Contains(err.Error(), "401") { + t.Fatalf("want 401 err, got %v", err) + } +} diff --git a/internal/tools/files.go b/internal/tools/files.go new file mode 100644 index 0000000..4b892b0 --- /dev/null +++ b/internal/tools/files.go @@ -0,0 +1,183 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/crypto/openclaw/internal/llm" +) + +// Workspace confines file access to a root directory using os.Root +// (Go 1.24+), which rejects "..", absolute paths and symlink escapes. +type Workspace struct { + Dir string + MaxBytes int64 +} + +func (w Workspace) open() (*os.Root, error) { + if err := os.MkdirAll(w.Dir, 0o750); err != nil { + return nil, err + } + return os.OpenRoot(w.Dir) +} + +func cleanRel(p string) (string, error) { + if p == "" { + p = "." + } + if filepath.IsAbs(p) { + return "", errors.New("path must be relative to the workspace") + } + return filepath.Clean(p), nil +} + +type pathArgs struct { + Path string `json:"path"` + Content string `json:"content"` +} + +// ---- file_read + +type FileRead struct{ Workspace } + +func (FileRead) Spec() llm.ToolSpec { + return llm.ToolSpec{Name: "file_read", Description: "Read a text file inside the workspace.", + Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}}, "path")} +} + +func (f FileRead) Call(_ context.Context, raw json.RawMessage) (string, error) { + var a pathArgs + if err := json.Unmarshal(raw, &a); err != nil { + return "", err + } + p, err := cleanRel(a.Path) + if err != nil { + return "", err + } + root, err := f.open() + if err != nil { + return "", err + } + defer root.Close() + fh, err := root.Open(p) + if err != nil { + return "", err + } + defer fh.Close() + b, err := io.ReadAll(io.LimitReader(fh, f.MaxBytes+1)) + if err != nil { + return "", err + } + if int64(len(b)) > f.MaxBytes { + return string(b[:f.MaxBytes]) + "\n[truncated]", nil + } + return string(b), nil +} + +// ---- file_write + +type FileWrite struct{ Workspace } + +func (FileWrite) Spec() llm.ToolSpec { + return llm.ToolSpec{Name: "file_write", Description: "Create or overwrite a text file inside the workspace.", + Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}, "content": map[string]any{"type": "string"}}, "path", "content")} +} + +func (f FileWrite) Call(_ context.Context, raw json.RawMessage) (string, error) { + var a pathArgs + if err := json.Unmarshal(raw, &a); err != nil { + return "", err + } + if int64(len(a.Content)) > f.MaxBytes { + return "", fmt.Errorf("content exceeds %d bytes", f.MaxBytes) + } + p, err := cleanRel(a.Path) + if err != nil { + return "", err + } + if p == "." { + return "", errors.New("path required") + } + root, err := f.open() + if err != nil { + return "", err + } + defer root.Close() + if dir := filepath.Dir(p); dir != "." { + if err := mkdirAll(root, dir); err != nil { + return "", err + } + } + fh, err := root.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640) + if err != nil { + return "", err + } + defer fh.Close() + if _, err := fh.WriteString(a.Content); err != nil { + return "", err + } + return fmt.Sprintf("wrote %d bytes to %s", len(a.Content), p), nil +} + +func mkdirAll(root *os.Root, dir string) error { + cur := "" + for _, part := range strings.Split(filepath.ToSlash(dir), "/") { + cur = filepath.Join(cur, part) + if err := root.Mkdir(cur, 0o750); err != nil && !errors.Is(err, os.ErrExist) { + return err + } + } + return nil +} + +// ---- file_list + +type FileList struct{ Workspace } + +func (FileList) Spec() llm.ToolSpec { + return llm.ToolSpec{Name: "file_list", Description: "List entries of a directory inside the workspace (default: root).", + Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}})} +} + +func (f FileList) Call(_ context.Context, raw json.RawMessage) (string, error) { + var a pathArgs + if err := json.Unmarshal(raw, &a); err != nil { + return "", err + } + p, err := cleanRel(a.Path) + if err != nil { + return "", err + } + root, err := f.open() + if err != nil { + return "", err + } + defer root.Close() + d, err := root.Open(p) + if err != nil { + return "", err + } + defer d.Close() + ents, err := d.ReadDir(500) + if err != nil && !errors.Is(err, io.EOF) { + return "", err + } + var sb strings.Builder + for _, e := range ents { + name := e.Name() + if e.IsDir() { + name += "/" + } + sb.WriteString(name + "\n") + } + if sb.Len() == 0 { + return "(empty)", nil + } + return sb.String(), nil +} diff --git a/internal/tools/timetool.go b/internal/tools/timetool.go new file mode 100644 index 0000000..f5569bc --- /dev/null +++ b/internal/tools/timetool.go @@ -0,0 +1,41 @@ +package tools + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/crypto/openclaw/internal/llm" +) + +// TimeNow returns the current time, optionally in an IANA time zone. +type TimeNow struct{ Now func() time.Time } + +func (TimeNow) Spec() llm.ToolSpec { + return llm.ToolSpec{ + Name: "time_now", + Description: "Get the current date and time. Optional IANA time zone, e.g. Europe/Berlin.", + Parameters: obj(map[string]any{"timezone": map[string]any{"type": "string"}}), + } +} + +func (t TimeNow) Call(_ context.Context, raw json.RawMessage) (string, error) { + var a struct{ Timezone string `json:"timezone"` } + if err := json.Unmarshal(raw, &a); err != nil { + return "", err + } + now := time.Now + if t.Now != nil { + now = t.Now + } + n := now() + if a.Timezone != "" { + loc, err := time.LoadLocation(a.Timezone) + if err != nil { + return "", fmt.Errorf("unknown timezone %q", a.Timezone) + } + n = n.In(loc) + } + return n.Format("2006-01-02T15:04:05 MST (Monday)"), nil +} diff --git a/internal/tools/tools.go b/internal/tools/tools.go new file mode 100644 index 0000000..0844916 --- /dev/null +++ b/internal/tools/tools.go @@ -0,0 +1,64 @@ +// Package tools contains the tool registry and built-in safe tools. +package tools + +import ( + "context" + "encoding/json" + "fmt" + "sort" + + "github.com/crypto/openclaw/internal/llm" +) + +type Tool interface { + Spec() llm.ToolSpec + Call(ctx context.Context, args json.RawMessage) (string, error) +} + +type Registry struct{ tools map[string]Tool } + +func NewRegistry(ts ...Tool) *Registry { + r := &Registry{tools: map[string]Tool{}} + for _, t := range ts { + r.Register(t) + } + return r +} + +func (r *Registry) Register(t Tool) { r.tools[t.Spec().Name] = t } + +func (r *Registry) Specs() []llm.ToolSpec { + out := make([]llm.ToolSpec, 0, len(r.tools)) + for _, t := range r.tools { + out = append(out, t.Spec()) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +// Call runs a tool; errors are returned as text so the model can react to them. +func (r *Registry) Call(ctx context.Context, name, args string) string { + t, ok := r.tools[name] + if !ok { + return fmt.Sprintf("error: unknown tool %q", name) + } + if args == "" { + args = "{}" + } + if !json.Valid([]byte(args)) { + return "error: arguments are not valid JSON" + } + res, err := t.Call(ctx, json.RawMessage(args)) + if err != nil { + return "error: " + err.Error() + } + return res +} + +func obj(props map[string]any, required ...string) map[string]any { + m := map[string]any{"type": "object", "properties": props} + if len(required) > 0 { + m["required"] = required + } + return m +} diff --git a/internal/tools/tools_test.go b/internal/tools/tools_test.go new file mode 100644 index 0000000..fa2adc8 --- /dev/null +++ b/internal/tools/tools_test.go @@ -0,0 +1,92 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestTimeNow(t *testing.T) { + fixed := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC) + out, err := TimeNow{Now: func() time.Time { return fixed }}.Call(context.Background(), json.RawMessage(`{"timezone":"Europe/Berlin"}`)) + if err != nil || !strings.HasPrefix(out, "2026-01-02T04:04:05") { + t.Fatalf("%q %v", out, err) + } + if _, err := (TimeNow{}).Call(context.Background(), json.RawMessage(`{"timezone":"Nope/Nope"}`)); err == nil { + t.Fatal("expected tz error") + } +} + +func TestWorkspaceConfinement(t *testing.T) { + dir := t.TempDir() + ws := Workspace{Dir: filepath.Join(dir, "ws"), MaxBytes: 100} + ctx := context.Background() + w, r, l := FileWrite{ws}, FileRead{ws}, FileList{ws} + if _, err := w.Call(ctx, json.RawMessage(`{"path":"a/b.txt","content":"hi"}`)); err != nil { + t.Fatal(err) + } + if out, err := r.Call(ctx, json.RawMessage(`{"path":"a/b.txt"}`)); err != nil || out != "hi" { + t.Fatalf("%q %v", out, err) + } + if out, _ := l.Call(ctx, json.RawMessage(`{}`)); out != "a/\n" { + t.Fatalf("list %q", out) + } + os.WriteFile(filepath.Join(dir, "secret"), []byte("x"), 0o600) + os.Symlink(filepath.Join(dir, "secret"), filepath.Join(ws.Dir, "link")) + for _, p := range []string{"../secret", "/etc/passwd", "a/../../secret", "link"} { + if _, err := r.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q}`, p))); err == nil { + t.Errorf("read of %q should fail", p) + } + if _, err := w.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q,"content":"x"}`, p))); err == nil { + t.Errorf("write of %q should fail", p) + } + } + if _, err := w.Call(ctx, json.RawMessage(`{"path":"big","content":"`+strings.Repeat("x", 101)+`"}`)); err == nil { + t.Error("size limit not enforced") + } +} + +func TestWebFetchBlocksPrivateByDefault(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "ok") })) + defer srv.Close() + _, err := NewWebFetch(2*time.Second, 1024, false).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`)) + if err == nil || !errors.Is(err, ErrBlockedAddr) { + t.Fatalf("expected blocked, got %v", err) + } + if _, err := NewWebFetch(time.Second, 10, false).Call(context.Background(), json.RawMessage(`{"url":"file:///etc/passwd"}`)); err == nil { + t.Fatal("file scheme must be rejected") + } +} + +func TestWebFetchStripsAndTruncates(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + fmt.Fprint(w, "

Hello World

"+strings.Repeat("y", 500)+"") + })) + defer srv.Close() + out, err := NewWebFetch(2*time.Second, 80, true).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "evil") || !strings.Contains(out, "Hello World") || !strings.HasSuffix(out, "[truncated]") { + t.Fatalf("unexpected: %q", out) + } +} + +func TestRegistryUnknownAndBadJSON(t *testing.T) { + r := NewRegistry(TimeNow{}) + if !strings.Contains(r.Call(context.Background(), "nope", "{}"), "unknown tool") { + t.Fatal() + } + if !strings.Contains(r.Call(context.Background(), "time_now", "{bad"), "not valid JSON") { + t.Fatal() + } +} diff --git a/internal/tools/webfetch.go b/internal/tools/webfetch.go new file mode 100644 index 0000000..1df9d25 --- /dev/null +++ b/internal/tools/webfetch.go @@ -0,0 +1,125 @@ +package tools + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "regexp" + "strings" + "syscall" + "time" + + "github.com/crypto/openclaw/internal/llm" +) + +// WebFetch downloads a URL with timeout and size limit. By default it refuses +// loopback/private/link-local targets (SSRF protection), checked at dial time +// so DNS rebinding and redirects are covered too. TLS verification is always on. +type WebFetch struct { + client *http.Client + maxBytes int64 +} + +var ErrBlockedAddr = errors.New("destination address is not allowed") + +func NewWebFetch(timeout time.Duration, maxBytes int64, allowPrivate bool) *WebFetch { + d := &net.Dialer{Timeout: 10 * time.Second} + if !allowPrivate { + d.Control = func(_, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return err + } + ip := net.ParseIP(host) + if ip == nil || blockedIP(ip) { + return ErrBlockedAddr + } + return nil + } + } + tr := &http.Transport{DialContext: d.DialContext, TLSHandshakeTimeout: 10 * time.Second, ForceAttemptHTTP2: true} + return &WebFetch{maxBytes: maxBytes, client: &http.Client{ + Timeout: timeout, Transport: tr, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("too many redirects") + } + if req.URL.Scheme != "http" && req.URL.Scheme != "https" { + return errors.New("redirect to unsupported scheme") + } + return nil + }, + }} +} + +func blockedIP(ip net.IP) bool { + return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || + ip.IsUnspecified() || ip.IsMulticast() || ip.IsInterfaceLocalMulticast() || + ip.Equal(net.IPv4bcast) || (ip.To4() != nil && ip.To4()[0] == 100 && ip.To4()[1]&0xC0 == 64) // CGNAT 100.64/10 +} + +func (w *WebFetch) Spec() llm.ToolSpec { + return llm.ToolSpec{ + Name: "web_fetch", + Description: "Fetch a public http(s) URL and return its text content (HTML is stripped, output truncated).", + Parameters: obj(map[string]any{"url": map[string]any{"type": "string"}}, "url"), + } +} + +var ( + reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?`) + reTag = regexp.MustCompile(`(?s)<[^>]+>`) + reSpace = regexp.MustCompile(`[ \t\r\f\v]+`) + reNL = regexp.MustCompile(`\n\s*\n+`) +) + +func (w *WebFetch) Call(ctx context.Context, raw json.RawMessage) (string, error) { + var a struct{ URL string `json:"url"` } + if err := json.Unmarshal(raw, &a); err != nil { + return "", err + } + u, err := url.Parse(a.URL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return "", fmt.Errorf("only absolute http(s) URLs are allowed") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", err + } + req.Header.Set("User-Agent", "OpenClaw/0.1 (+self-hosted assistant)") + resp, err := w.client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, w.maxBytes+1)) + if err != nil { + return "", err + } + truncated := int64(len(body)) > w.maxBytes + if truncated { + body = body[:w.maxBytes] + } + text := string(body) + if strings.Contains(resp.Header.Get("Content-Type"), "html") { + text = reScript.ReplaceAllString(text, "") + text = reTag.ReplaceAllString(text, " ") + text = reSpace.ReplaceAllString(text, " ") + text = reNL.ReplaceAllString(text, "\n\n") + } + text = strings.TrimSpace(text) + const maxOut = 20000 + if len(text) > maxOut { + text, truncated = text[:maxOut], true + } + out := fmt.Sprintf("HTTP %d %s\n\n%s", resp.StatusCode, resp.Header.Get("Content-Type"), text) + if truncated { + out += "\n\n[truncated]" + } + return out, nil +}