feat: config, OpenAI-compatible streaming provider and safe tools

internal/tools/files.go

@@ -0,0 +1,183 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/crypto/openclaw/internal/llm"
+)
+
+// Workspace confines file access to a root directory using os.Root
+// (Go 1.24+), which rejects "..", absolute paths and symlink escapes.
+type Workspace struct {
+	Dir      string
+	MaxBytes int64
+}
+
+func (w Workspace) open() (*os.Root, error) {
+	if err := os.MkdirAll(w.Dir, 0o750); err != nil {
+		return nil, err
+	}
+	return os.OpenRoot(w.Dir)
+}
+
+func cleanRel(p string) (string, error) {
+	if p == "" {
+		p = "."
+	}
+	if filepath.IsAbs(p) {
+		return "", errors.New("path must be relative to the workspace")
+	}
+	return filepath.Clean(p), nil
+}
+
+type pathArgs struct {
+	Path    string `json:"path"`
+	Content string `json:"content"`
+}
+
+// ---- file_read
+
+type FileRead struct{ Workspace }
+
+func (FileRead) Spec() llm.ToolSpec {
+	return llm.ToolSpec{Name: "file_read", Description: "Read a text file inside the workspace.",
+		Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}}, "path")}
+}
+
+func (f FileRead) Call(_ context.Context, raw json.RawMessage) (string, error) {
+	var a pathArgs
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return "", err
+	}
+	p, err := cleanRel(a.Path)
+	if err != nil {
+		return "", err
+	}
+	root, err := f.open()
+	if err != nil {
+		return "", err
+	}
+	defer root.Close()
+	fh, err := root.Open(p)
+	if err != nil {
+		return "", err
+	}
+	defer fh.Close()
+	b, err := io.ReadAll(io.LimitReader(fh, f.MaxBytes+1))
+	if err != nil {
+		return "", err
+	}
+	if int64(len(b)) > f.MaxBytes {
+		return string(b[:f.MaxBytes]) + "\n[truncated]", nil
+	}
+	return string(b), nil
+}
+
+// ---- file_write
+
+type FileWrite struct{ Workspace }
+
+func (FileWrite) Spec() llm.ToolSpec {
+	return llm.ToolSpec{Name: "file_write", Description: "Create or overwrite a text file inside the workspace.",
+		Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}, "content": map[string]any{"type": "string"}}, "path", "content")}
+}
+
+func (f FileWrite) Call(_ context.Context, raw json.RawMessage) (string, error) {
+	var a pathArgs
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return "", err
+	}
+	if int64(len(a.Content)) > f.MaxBytes {
+		return "", fmt.Errorf("content exceeds %d bytes", f.MaxBytes)
+	}
+	p, err := cleanRel(a.Path)
+	if err != nil {
+		return "", err
+	}
+	if p == "." {
+		return "", errors.New("path required")
+	}
+	root, err := f.open()
+	if err != nil {
+		return "", err
+	}
+	defer root.Close()
+	if dir := filepath.Dir(p); dir != "." {
+		if err := mkdirAll(root, dir); err != nil {
+			return "", err
+		}
+	}
+	fh, err := root.OpenFile(p, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o640)
+	if err != nil {
+		return "", err
+	}
+	defer fh.Close()
+	if _, err := fh.WriteString(a.Content); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("wrote %d bytes to %s", len(a.Content), p), nil
+}
+
+func mkdirAll(root *os.Root, dir string) error {
+	cur := ""
+	for _, part := range strings.Split(filepath.ToSlash(dir), "/") {
+		cur = filepath.Join(cur, part)
+		if err := root.Mkdir(cur, 0o750); err != nil && !errors.Is(err, os.ErrExist) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ---- file_list
+
+type FileList struct{ Workspace }
+
+func (FileList) Spec() llm.ToolSpec {
+	return llm.ToolSpec{Name: "file_list", Description: "List entries of a directory inside the workspace (default: root).",
+		Parameters: obj(map[string]any{"path": map[string]any{"type": "string"}})}
+}
+
+func (f FileList) Call(_ context.Context, raw json.RawMessage) (string, error) {
+	var a pathArgs
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return "", err
+	}
+	p, err := cleanRel(a.Path)
+	if err != nil {
+		return "", err
+	}
+	root, err := f.open()
+	if err != nil {
+		return "", err
+	}
+	defer root.Close()
+	d, err := root.Open(p)
+	if err != nil {
+		return "", err
+	}
+	defer d.Close()
+	ents, err := d.ReadDir(500)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return "", err
+	}
+	var sb strings.Builder
+	for _, e := range ents {
+		name := e.Name()
+		if e.IsDir() {
+			name += "/"
+		}
+		sb.WriteString(name + "\n")
+	}
+	if sb.Len() == 0 {
+		return "(empty)", nil
+	}
+	return sb.String(), nil
+}

internal/tools/timetool.go

@@ -0,0 +1,41 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"github.com/crypto/openclaw/internal/llm"
+)
+
+// TimeNow returns the current time, optionally in an IANA time zone.
+type TimeNow struct{ Now func() time.Time }
+
+func (TimeNow) Spec() llm.ToolSpec {
+	return llm.ToolSpec{
+		Name:        "time_now",
+		Description: "Get the current date and time. Optional IANA time zone, e.g. Europe/Berlin.",
+		Parameters:  obj(map[string]any{"timezone": map[string]any{"type": "string"}}),
+	}
+}
+
+func (t TimeNow) Call(_ context.Context, raw json.RawMessage) (string, error) {
+	var a struct{ Timezone string `json:"timezone"` }
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return "", err
+	}
+	now := time.Now
+	if t.Now != nil {
+		now = t.Now
+	}
+	n := now()
+	if a.Timezone != "" {
+		loc, err := time.LoadLocation(a.Timezone)
+		if err != nil {
+			return "", fmt.Errorf("unknown timezone %q", a.Timezone)
+		}
+		n = n.In(loc)
+	}
+	return n.Format("2006-01-02T15:04:05 MST (Monday)"), nil
+}

internal/tools/tools.go

@@ -0,0 +1,64 @@
+// Package tools contains the tool registry and built-in safe tools.
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sort"
+
+	"github.com/crypto/openclaw/internal/llm"
+)
+
+type Tool interface {
+	Spec() llm.ToolSpec
+	Call(ctx context.Context, args json.RawMessage) (string, error)
+}
+
+type Registry struct{ tools map[string]Tool }
+
+func NewRegistry(ts ...Tool) *Registry {
+	r := &Registry{tools: map[string]Tool{}}
+	for _, t := range ts {
+		r.Register(t)
+	}
+	return r
+}
+
+func (r *Registry) Register(t Tool) { r.tools[t.Spec().Name] = t }
+
+func (r *Registry) Specs() []llm.ToolSpec {
+	out := make([]llm.ToolSpec, 0, len(r.tools))
+	for _, t := range r.tools {
+		out = append(out, t.Spec())
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+	return out
+}
+
+// Call runs a tool; errors are returned as text so the model can react to them.
+func (r *Registry) Call(ctx context.Context, name, args string) string {
+	t, ok := r.tools[name]
+	if !ok {
+		return fmt.Sprintf("error: unknown tool %q", name)
+	}
+	if args == "" {
+		args = "{}"
+	}
+	if !json.Valid([]byte(args)) {
+		return "error: arguments are not valid JSON"
+	}
+	res, err := t.Call(ctx, json.RawMessage(args))
+	if err != nil {
+		return "error: " + err.Error()
+	}
+	return res
+}
+
+func obj(props map[string]any, required ...string) map[string]any {
+	m := map[string]any{"type": "object", "properties": props}
+	if len(required) > 0 {
+		m["required"] = required
+	}
+	return m
+}

internal/tools/tools_test.go

@@ -0,0 +1,92 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestTimeNow(t *testing.T) {
+	fixed := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
+	out, err := TimeNow{Now: func() time.Time { return fixed }}.Call(context.Background(), json.RawMessage(`{"timezone":"Europe/Berlin"}`))
+	if err != nil || !strings.HasPrefix(out, "2026-01-02T04:04:05") {
+		t.Fatalf("%q %v", out, err)
+	}
+	if _, err := (TimeNow{}).Call(context.Background(), json.RawMessage(`{"timezone":"Nope/Nope"}`)); err == nil {
+		t.Fatal("expected tz error")
+	}
+}
+
+func TestWorkspaceConfinement(t *testing.T) {
+	dir := t.TempDir()
+	ws := Workspace{Dir: filepath.Join(dir, "ws"), MaxBytes: 100}
+	ctx := context.Background()
+	w, r, l := FileWrite{ws}, FileRead{ws}, FileList{ws}
+	if _, err := w.Call(ctx, json.RawMessage(`{"path":"a/b.txt","content":"hi"}`)); err != nil {
+		t.Fatal(err)
+	}
+	if out, err := r.Call(ctx, json.RawMessage(`{"path":"a/b.txt"}`)); err != nil || out != "hi" {
+		t.Fatalf("%q %v", out, err)
+	}
+	if out, _ := l.Call(ctx, json.RawMessage(`{}`)); out != "a/\n" {
+		t.Fatalf("list %q", out)
+	}
+	os.WriteFile(filepath.Join(dir, "secret"), []byte("x"), 0o600)
+	os.Symlink(filepath.Join(dir, "secret"), filepath.Join(ws.Dir, "link"))
+	for _, p := range []string{"../secret", "/etc/passwd", "a/../../secret", "link"} {
+		if _, err := r.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q}`, p))); err == nil {
+			t.Errorf("read of %q should fail", p)
+		}
+		if _, err := w.Call(ctx, json.RawMessage(fmt.Sprintf(`{"path":%q,"content":"x"}`, p))); err == nil {
+			t.Errorf("write of %q should fail", p)
+		}
+	}
+	if _, err := w.Call(ctx, json.RawMessage(`{"path":"big","content":"`+strings.Repeat("x", 101)+`"}`)); err == nil {
+		t.Error("size limit not enforced")
+	}
+}
+
+func TestWebFetchBlocksPrivateByDefault(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, "ok") }))
+	defer srv.Close()
+	_, err := NewWebFetch(2*time.Second, 1024, false).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`))
+	if err == nil || !errors.Is(err, ErrBlockedAddr) {
+		t.Fatalf("expected blocked, got %v", err)
+	}
+	if _, err := NewWebFetch(time.Second, 10, false).Call(context.Background(), json.RawMessage(`{"url":"file:///etc/passwd"}`)); err == nil {
+		t.Fatal("file scheme must be rejected")
+	}
+}
+
+func TestWebFetchStripsAndTruncates(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/html")
+		fmt.Fprint(w, "<html><script>evil()</script><p>Hello <b>World</b></p>"+strings.Repeat("y", 500)+"</html>")
+	}))
+	defer srv.Close()
+	out, err := NewWebFetch(2*time.Second, 80, true).Call(context.Background(), json.RawMessage(`{"url":"`+srv.URL+`"}`))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if strings.Contains(out, "evil") || !strings.Contains(out, "Hello World") || !strings.HasSuffix(out, "[truncated]") {
+		t.Fatalf("unexpected: %q", out)
+	}
+}
+
+func TestRegistryUnknownAndBadJSON(t *testing.T) {
+	r := NewRegistry(TimeNow{})
+	if !strings.Contains(r.Call(context.Background(), "nope", "{}"), "unknown tool") {
+		t.Fatal()
+	}
+	if !strings.Contains(r.Call(context.Background(), "time_now", "{bad"), "not valid JSON") {
+		t.Fatal()
+	}
+}

internal/tools/webfetch.go

@@ -0,0 +1,125 @@
+package tools
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/crypto/openclaw/internal/llm"
+)
+
+// WebFetch downloads a URL with timeout and size limit. By default it refuses
+// loopback/private/link-local targets (SSRF protection), checked at dial time
+// so DNS rebinding and redirects are covered too. TLS verification is always on.
+type WebFetch struct {
+	client   *http.Client
+	maxBytes int64
+}
+
+var ErrBlockedAddr = errors.New("destination address is not allowed")
+
+func NewWebFetch(timeout time.Duration, maxBytes int64, allowPrivate bool) *WebFetch {
+	d := &net.Dialer{Timeout: 10 * time.Second}
+	if !allowPrivate {
+		d.Control = func(_, address string, _ syscall.RawConn) error {
+			host, _, err := net.SplitHostPort(address)
+			if err != nil {
+				return err
+			}
+			ip := net.ParseIP(host)
+			if ip == nil || blockedIP(ip) {
+				return ErrBlockedAddr
+			}
+			return nil
+		}
+	}
+	tr := &http.Transport{DialContext: d.DialContext, TLSHandshakeTimeout: 10 * time.Second, ForceAttemptHTTP2: true}
+	return &WebFetch{maxBytes: maxBytes, client: &http.Client{
+		Timeout: timeout, Transport: tr,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			if len(via) >= 5 {
+				return errors.New("too many redirects")
+			}
+			if req.URL.Scheme != "http" && req.URL.Scheme != "https" {
+				return errors.New("redirect to unsupported scheme")
+			}
+			return nil
+		},
+	}}
+}
+
+func blockedIP(ip net.IP) bool {
+	return ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
+		ip.IsUnspecified() || ip.IsMulticast() || ip.IsInterfaceLocalMulticast() ||
+		ip.Equal(net.IPv4bcast) || (ip.To4() != nil && ip.To4()[0] == 100 && ip.To4()[1]&0xC0 == 64) // CGNAT 100.64/10
+}
+
+func (w *WebFetch) Spec() llm.ToolSpec {
+	return llm.ToolSpec{
+		Name:        "web_fetch",
+		Description: "Fetch a public http(s) URL and return its text content (HTML is stripped, output truncated).",
+		Parameters:  obj(map[string]any{"url": map[string]any{"type": "string"}}, "url"),
+	}
+}
+
+var (
+	reScript = regexp.MustCompile(`(?is)<(script|style|noscript)[^>]*>.*?</(script|style|noscript)>`)
+	reTag    = regexp.MustCompile(`(?s)<[^>]+>`)
+	reSpace  = regexp.MustCompile(`[ \t\r\f\v]+`)
+	reNL     = regexp.MustCompile(`\n\s*\n+`)
+)
+
+func (w *WebFetch) Call(ctx context.Context, raw json.RawMessage) (string, error) {
+	var a struct{ URL string `json:"url"` }
+	if err := json.Unmarshal(raw, &a); err != nil {
+		return "", err
+	}
+	u, err := url.Parse(a.URL)
+	if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
+		return "", fmt.Errorf("only absolute http(s) URLs are allowed")
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("User-Agent", "OpenClaw/0.1 (+self-hosted assistant)")
+	resp, err := w.client.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, w.maxBytes+1))
+	if err != nil {
+		return "", err
+	}
+	truncated := int64(len(body)) > w.maxBytes
+	if truncated {
+		body = body[:w.maxBytes]
+	}
+	text := string(body)
+	if strings.Contains(resp.Header.Get("Content-Type"), "html") {
+		text = reScript.ReplaceAllString(text, "")
+		text = reTag.ReplaceAllString(text, " ")
+		text = reSpace.ReplaceAllString(text, " ")
+		text = reNL.ReplaceAllString(text, "\n\n")
+	}
+	text = strings.TrimSpace(text)
+	const maxOut = 20000
+	if len(text) > maxOut {
+		text, truncated = text[:maxOut], true
+	}
+	out := fmt.Sprintf("HTTP %d %s\n\n%s", resp.StatusCode, resp.Header.Get("Content-Type"), text)
+	if truncated {
+		out += "\n\n[truncated]"
+	}
+	return out, nil
+}