feat: provider manager, cancellation, session summaries (store+agent)
This commit is contained in:
OpenClaw Dev
parent
406a523d54
commit
f1b2093b22
10 changed files with 883 additions and 37 deletions
255
internal/providers/providers.go
Normal file
255
internal/providers/providers.go
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
// Package providers manages the set of configured LLM providers and which one
|
||||
// is active (globally or per session). API keys never leave this package.
|
||||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/crypto/byteclaw/internal/config"
|
||||
"github.com/crypto/byteclaw/internal/llm"
|
||||
"github.com/crypto/byteclaw/internal/store"
|
||||
)
|
||||
|
||||
var ErrUnknown = errors.New("unknown provider")
|
||||
var ErrNotConfigured = errors.New("provider has no API key configured")
|
||||
|
||||
const settingKey = "active_provider"
|
||||
|
||||
type entry struct {
|
||||
name, baseURL, model, apiKey string
|
||||
keyless bool
|
||||
}
|
||||
|
||||
// Info is the public view of a provider (no secrets).
|
||||
type Info struct {
|
||||
Name string `json:"name"`
|
||||
BaseURL string `json:"base_url"`
|
||||
DefaultModel string `json:"default_model"`
|
||||
Configured bool `json:"configured"`
|
||||
Active bool `json:"active"`
|
||||
}
|
||||
|
||||
type Active struct {
|
||||
Provider string `json:"provider"`
|
||||
Model string `json:"model"`
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
mu sync.RWMutex
|
||||
entries map[string]*entry
|
||||
active Active
|
||||
store *store.Store
|
||||
timeout time.Duration
|
||||
temperature float64
|
||||
HTTP *http.Client
|
||||
// New builds a provider; replaceable in tests.
|
||||
New func(baseURL, apiKey, model string) llm.Provider
|
||||
}
|
||||
|
||||
// New builds the manager from config: every preset (key from llm.providers.<name>
|
||||
// or the preset's env var) plus the provider configured in llm.*.
|
||||
func New(cfg config.LLM, getenv func(string) string, st *store.Store) *Manager {
|
||||
m := &Manager{entries: map[string]*entry{}, store: st, timeout: time.Duration(cfg.TimeoutSec) * time.Second,
|
||||
temperature: cfg.Temperature, HTTP: &http.Client{Timeout: 15 * time.Second}}
|
||||
m.New = func(baseURL, apiKey, model string) llm.Provider {
|
||||
return llm.NewOpenAI(baseURL, apiKey, model, m.temperature, m.timeout)
|
||||
}
|
||||
for name, p := range config.Presets {
|
||||
e := &entry{name: name, baseURL: p.BaseURL, model: p.DefaultModel, keyless: p.KeyEnv == ""}
|
||||
if p.KeyEnv != "" {
|
||||
e.apiKey = getenv(p.KeyEnv)
|
||||
}
|
||||
m.entries[name] = e
|
||||
}
|
||||
for name, o := range cfg.Providers {
|
||||
name = strings.ToLower(name)
|
||||
e, ok := m.entries[name]
|
||||
if !ok {
|
||||
e = &entry{name: name}
|
||||
m.entries[name] = e
|
||||
}
|
||||
if o.BaseURL != "" {
|
||||
e.baseURL = o.BaseURL
|
||||
}
|
||||
if o.APIKey != "" {
|
||||
e.apiKey = o.APIKey
|
||||
}
|
||||
if o.Model != "" {
|
||||
e.model = o.Model
|
||||
}
|
||||
}
|
||||
// The primary llm.* settings win for the configured provider.
|
||||
e, ok := m.entries[cfg.Provider]
|
||||
if !ok {
|
||||
e = &entry{name: cfg.Provider, keyless: true}
|
||||
m.entries[cfg.Provider] = e
|
||||
}
|
||||
e.baseURL, e.model = cfg.BaseURL, cfg.Model
|
||||
if cfg.APIKey != "" {
|
||||
e.apiKey = cfg.APIKey
|
||||
}
|
||||
m.active = Active{Provider: cfg.Provider, Model: cfg.Model}
|
||||
if st != nil {
|
||||
if raw, err := st.GetSetting(context.Background(), settingKey); err == nil {
|
||||
var a Active
|
||||
if json.Unmarshal([]byte(raw), &a) == nil && m.usable(a.Provider) == nil {
|
||||
m.active = a
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (m *Manager) usable(name string) error {
|
||||
e, ok := m.entries[name]
|
||||
if !ok || e.baseURL == "" {
|
||||
return fmt.Errorf("%w: %q", ErrUnknown, name)
|
||||
}
|
||||
if e.apiKey == "" && !e.keyless {
|
||||
return fmt.Errorf("%w: %q", ErrNotConfigured, name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *Manager) List() []Info {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
out := make([]Info, 0, len(m.entries))
|
||||
for _, e := range m.entries {
|
||||
if e.baseURL == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, Info{Name: e.name, BaseURL: e.baseURL, DefaultModel: e.model,
|
||||
Configured: m.usable(e.name) == nil, Active: e.name == m.active.Provider})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
|
||||
func (m *Manager) Active() Active {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.active
|
||||
}
|
||||
|
||||
func (m *Manager) BaseURL(name string) string {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
if e, ok := m.entries[name]; ok {
|
||||
return e.baseURL
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Validate checks provider/model and fills the default model.
|
||||
func (m *Manager) Validate(name, model string) (Active, error) {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
name = strings.ToLower(strings.TrimSpace(name))
|
||||
if err := m.usable(name); err != nil {
|
||||
return Active{}, err
|
||||
}
|
||||
if model = strings.TrimSpace(model); model == "" {
|
||||
model = m.entries[name].model
|
||||
}
|
||||
if model == "" {
|
||||
return Active{}, errors.New("model required")
|
||||
}
|
||||
return Active{Provider: name, Model: model}, nil
|
||||
}
|
||||
|
||||
// SetActive switches the server-wide default and persists it.
|
||||
func (m *Manager) SetActive(ctx context.Context, name, model string) (Active, error) {
|
||||
a, err := m.Validate(name, model)
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.active = a
|
||||
m.mu.Unlock()
|
||||
if m.store != nil {
|
||||
b, _ := json.Marshal(a)
|
||||
if err := m.store.SetSetting(ctx, settingKey, string(b)); err != nil {
|
||||
return a, err
|
||||
}
|
||||
}
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// Build returns a provider client for name/model.
|
||||
func (m *Manager) Build(name, model string) (llm.Provider, error) {
|
||||
a, err := m.Validate(name, model)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.mu.RLock()
|
||||
e := *m.entries[a.Provider]
|
||||
m.mu.RUnlock()
|
||||
return m.New(e.baseURL, e.apiKey, a.Model), nil
|
||||
}
|
||||
|
||||
// ForSession resolves the provider for a session (session override, else active).
|
||||
func (m *Manager) ForSession(ctx context.Context, sessionID string) (llm.Provider, Active, error) {
|
||||
a := m.Active()
|
||||
if m.store != nil && sessionID != "" {
|
||||
if ss, err := m.store.GetSession(ctx, sessionID); err == nil && ss.Provider != "" {
|
||||
a = Active{Provider: ss.Provider, Model: ss.Model}
|
||||
}
|
||||
}
|
||||
p, err := m.Build(a.Provider, a.Model)
|
||||
return p, a, err
|
||||
}
|
||||
|
||||
// Models lists model ids from the provider's OpenAI-compatible GET /models (key added server-side).
|
||||
func (m *Manager) Models(ctx context.Context, name string) ([]string, error) {
|
||||
m.mu.RLock()
|
||||
e, ok := m.entries[strings.ToLower(name)]
|
||||
var ec entry
|
||||
if ok {
|
||||
ec = *e
|
||||
}
|
||||
m.mu.RUnlock()
|
||||
if !ok || ec.baseURL == "" {
|
||||
return nil, fmt.Errorf("%w: %q", ErrUnknown, name)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(ec.baseURL, "/")+"/models", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ec.apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+ec.apiKey)
|
||||
}
|
||||
resp, err := m.HTTP.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, fmt.Errorf("models endpoint returned HTTP %d", resp.StatusCode)
|
||||
}
|
||||
var body struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&body); err != nil {
|
||||
return nil, fmt.Errorf("decode models: %w", err)
|
||||
}
|
||||
ids := make([]string, 0, len(body.Data))
|
||||
for _, d := range body.Data {
|
||||
if d.ID != "" {
|
||||
ids = append(ids, d.ID)
|
||||
}
|
||||
}
|
||||
sort.Strings(ids)
|
||||
return ids, nil
|
||||
}
|
||||
100
internal/providers/providers_test.go
Normal file
100
internal/providers/providers_test.go
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
package providers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/crypto/byteclaw/internal/config"
|
||||
"github.com/crypto/byteclaw/internal/llm"
|
||||
"github.com/crypto/byteclaw/internal/store"
|
||||
)
|
||||
|
||||
type fake struct{ base, key, model string }
|
||||
|
||||
func (fake) Chat(context.Context, llm.Request, llm.DeltaFunc) (llm.Response, error) {
|
||||
return llm.Response{}, nil
|
||||
}
|
||||
|
||||
func newMgr(t *testing.T, env map[string]string) (*Manager, *store.Store) {
|
||||
st, err := store.Open(filepath.Join(t.TempDir(), "p.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
cfg := config.LLM{Provider: "openai", BaseURL: "https://api.openai.com/v1", Model: "gpt-4o-mini", APIKey: "sk-openai",
|
||||
Providers: map[string]config.ProviderOverride{"gonkarouter": {APIKey: "sk-gonka"}}}
|
||||
m := New(cfg, func(k string) string { return env[k] }, st)
|
||||
m.New = func(b, k, mo string) llm.Provider { return fake{b, k, mo} }
|
||||
return m, st
|
||||
}
|
||||
|
||||
func TestListAndSwitch(t *testing.T) {
|
||||
m, st := newMgr(t, map[string]string{"DAHL_API_KEY": "dahl_k"})
|
||||
got := map[string]Info{}
|
||||
for _, i := range m.List() {
|
||||
got[i.Name] = i
|
||||
}
|
||||
if !got["openai"].Active || !got["dahl"].Configured || !got["gonkarouter"].Configured || got["openrouter"].Configured || !got["ollama"].Configured {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
if _, err := m.SetActive(context.Background(), "openrouter", ""); !errors.Is(err, ErrNotConfigured) {
|
||||
t.Fatalf("want not configured, got %v", err)
|
||||
}
|
||||
if _, err := m.SetActive(context.Background(), "nope", ""); !errors.Is(err, ErrUnknown) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a, err := m.SetActive(context.Background(), "Dahl", "")
|
||||
if err != nil || a.Provider != "dahl" || a.Model != "MiniMaxAI/MiniMax-M2.7" {
|
||||
t.Fatalf("%+v %v", a, err)
|
||||
}
|
||||
p, _ := m.Build("dahl", "zai-org/GLM-5.3-Flash")
|
||||
if f := p.(fake); f.key != "dahl_k" || f.base != "https://inference.dahl.global/v1" || f.model != "zai-org/GLM-5.3-Flash" {
|
||||
t.Fatalf("%+v", f)
|
||||
}
|
||||
// persisted across restarts
|
||||
m2 := New(config.LLM{Provider: "openai", BaseURL: "x", Model: "y"}, func(k string) string { return map[string]string{"DAHL_API_KEY": "dahl_k"}[k] }, st)
|
||||
if m2.Active().Provider != "dahl" {
|
||||
t.Fatalf("not persisted: %+v", m2.Active())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionOverride(t *testing.T) {
|
||||
m, st := newMgr(t, nil)
|
||||
ctx := context.Background()
|
||||
ss, _ := st.CreateSession(ctx, "")
|
||||
st.SetSessionProvider(ctx, ss.ID, "gonkarouter", "MiniMaxAI/MiniMax-M2.7")
|
||||
p, a, err := m.ForSession(ctx, ss.ID)
|
||||
if err != nil || a.Provider != "gonkarouter" || p.(fake).key != "sk-gonka" {
|
||||
t.Fatalf("%+v %v", a, err)
|
||||
}
|
||||
other, _ := st.CreateSession(ctx, "")
|
||||
if _, a, _ := m.ForSession(ctx, other.ID); a.Provider != "openai" {
|
||||
t.Fatalf("%+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelsUsesServerSideKey(t *testing.T) {
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/v1/models" || r.Header.Get("Authorization") != "Bearer sk-gonka" {
|
||||
http.Error(w, "no", 401)
|
||||
return
|
||||
}
|
||||
fmt.Fprint(w, `{"data":[{"id":"b-model"},{"id":"a-model"}]}`)
|
||||
}))
|
||||
defer srv.Close()
|
||||
m, _ := newMgr(t, nil)
|
||||
m.entries["gonkarouter"].baseURL = srv.URL + "/v1"
|
||||
ids, err := m.Models(context.Background(), "gonkarouter")
|
||||
if err != nil || len(ids) != 2 || ids[0] != "a-model" {
|
||||
t.Fatalf("%v %v", ids, err)
|
||||
}
|
||||
m.entries["gonkarouter"].apiKey = "wrong"
|
||||
if _, err := m.Models(context.Background(), "gonkarouter"); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue