diff --git a/cmd/livepeer/integration_test.go b/cmd/livepeer/integration_test.go index ef65bf9..4d0cce1 100644 --- a/cmd/livepeer/integration_test.go +++ b/cmd/livepeer/integration_test.go @@ -2,6 +2,7 @@ package main import ( "bytes" + "context" "crypto/sha256" "encoding/json" "fmt" @@ -91,6 +92,7 @@ func TestRealBinaryOffchainFlow(t *testing.T) { } testBinaryMigrations(t, install) + t.Run("discovery dump", func(t *testing.T) { testDiscoveryDumpBinary(t, install) }) runner := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { require.NotEmpty(t, r.Header.Get("Livepeer-Session-Token")) @@ -233,6 +235,161 @@ func testDiscoveryBinary(t *testing.T, install, orchestratorURL string) { }, 5*time.Second, 20*time.Millisecond) } +func testDiscoveryDumpBinary(t *testing.T, install string) { + t.Helper() + listen, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listen.Close() + metrics, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer metrics.Close() + started := make(chan error, 1) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/inventory/discovery": + _, _ = io.WriteString(w, `[{"address":"https://orch.example","runners":[{"url":"https://orch.example/apps/runner/app","app":"test","observed_at":"2000-01-01T00:00:00Z"}]}]`) + case "/blocked/a/discovery", "/blocked/b/discovery": + _, _ = fmt.Fprintf(w, `[{"address":"https://orch.example","runners":[{"url":%q,"app":"test","observed_at":"2000-01-01T00:00:00Z","extension":%q}]}]`, "https://orch.example"+r.URL.Path, strings.Repeat("x", 900<<10)) + case "/failed/discovery": + w.WriteHeader(http.StatusServiceUnavailable) + case "/cancel/discovery": + started <- nil + <-r.Context().Done() + default: + t.Errorf("unexpected request: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + + for _, entry := range [][]string{{"livepeer-discovery"}, {"livepeer", "discovery"}} { + t.Run(entry[0], func(t *testing.T) { + for _, tc := range []struct { + name string + sources []string + flags []string + entries int + exit int + diagnostic string + noOutput bool + }{ + {name: "inventory", sources: []string{"inventory"}, entries: 2}, + {name: "partial failure", sources: []string{"inventory", "failed"}, entries: 2, diagnostic: "discovery refresh failed"}, + {name: "total failure", sources: []string{"failed"}, exit: 2, diagnostic: "all discovery sources failed"}, + {name: "conflicting flags", flags: []string{"--print-config"}, exit: 2, diagnostic: "mutually exclusive", noOutput: true}, + {name: "broken stdout", exit: 2, diagnostic: "cannot write discovery inventory", noOutput: true}, + {name: "blocked stdout", sources: []string{"blocked/a", "blocked/b"}, exit: 2, diagnostic: "cannot write discovery inventory", noOutput: true}, + {name: "reader closes after cancellation", sources: []string{"blocked/a"}, exit: 2, diagnostic: "cannot write discovery inventory", noOutput: true}, + {name: "cancellation", sources: []string{"cancel"}, exit: 2, diagnostic: "context canceled", noOutput: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + args := append([]string{}, entry[1:]...) + args = append(args, "--config=", "--dump", "--listen", listen.Addr().String(), "--metrics-listen", metrics.Addr().String(), + "--grants", strings.TrimPrefix(upstream.URL, "http://")) + for _, source := range tc.sources { + args = append(args, "--nodes", upstream.URL+"/"+source) + } + args = append(args, tc.flags...) + cmd := exec.CommandContext(ctx, filepath.Join(install, entry[0]), args...) + var out, logs bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &logs + var ready chan error + if tc.name == "cancellation" { + ready = started + } + var stdoutReader, stderrReader, stderrWriter *os.File + if tc.name == "broken stdout" || tc.name == "blocked stdout" || tc.name == "reader closes after cancellation" { + reader, writer, err := os.Pipe() + require.NoError(t, err) + defer reader.Close() + defer writer.Close() + cmd.Stdout = writer + stdoutReader = reader + if tc.name == "broken stdout" { + require.NoError(t, reader.Close()) + } else { + ready = make(chan error, 1) + go func() { + var first [1]byte + _, err := reader.Read(first[:]) + ready <- err + }() + } + } + if tc.name == "reader closes after cancellation" { + stderrReader, stderrWriter, err = os.Pipe() + require.NoError(t, err) + defer stderrReader.Close() + defer stderrWriter.Close() + require.NoError(t, stderrWriter.SetWriteDeadline(time.Now().Add(100*time.Millisecond))) + _, err := stderrWriter.Write(make([]byte, 1<<20)) + require.ErrorIs(t, err, os.ErrDeadlineExceeded) + require.NoError(t, stderrWriter.SetWriteDeadline(time.Time{})) + cmd.Stderr = stderrWriter + } + require.NoError(t, cmd.Start()) + if stderrWriter != nil { + require.NoError(t, stderrWriter.Close()) + } + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + if ready != nil { + select { + case err := <-ready: + require.NoError(t, err) + require.NoError(t, cmd.Process.Signal(syscall.SIGTERM)) + case <-ctx.Done(): + t.Fatal("dump did not become ready for cancellation") + } + } + if stderrReader != nil { + // Delay the final error report until cancellation has returned + // from Root and the stdout reader has closed. + for i := range 2 { + select { + case err := <-done: + t.Fatalf("dump exited before stderr was drained: %v", err) + case <-time.After(100 * time.Millisecond): + } + if i == 0 { + require.NoError(t, stdoutReader.Close()) + } + } + _, err := io.Copy(&logs, stderrReader) + require.NoError(t, err) + } + err := <-done + require.NoError(t, ctx.Err(), "dump must finish before its process deadline") + if tc.exit == 0 { + require.NoError(t, err, logs.String()) + } else { + var exit *exec.ExitError + require.ErrorAs(t, err, &exit, logs.String()) + require.Equal(t, tc.exit, exit.ExitCode(), logs.String()) + } + if tc.diagnostic == "" { + require.Empty(t, logs.String()) + } else { + require.Contains(t, logs.String(), tc.diagnostic) + } + if tc.noOutput { + require.Empty(t, out.String()) + return + } + var entries []json.RawMessage + require.NoError(t, json.Unmarshal(out.Bytes(), &entries), out.String()) + require.NotNil(t, entries) + require.Len(t, entries, tc.entries) + require.True(t, strings.HasSuffix(out.String(), "\n")) + require.Equal(t, 1, strings.Count(out.String(), "\n")) + }) + } + }) + } +} + func checkPythonSDK(t *testing.T, root, serviceURL, runnerURL string) { t.Helper() python, export := test.PythonSDK(t, root) diff --git a/cmd/livepeer/testdata/cli/discovery-help.golden b/cmd/livepeer/testdata/cli/discovery-help.golden index 084419d..e7b0346 100644 --- a/cmd/livepeer/testdata/cli/discovery-help.golden +++ b/cmd/livepeer/testdata/cli/discovery-help.golden @@ -16,5 +16,6 @@ Flags: --max-failed-refreshes int Consecutive failed refreshes before eviction (env: LIVEPEER_DISCOVERY_MAX_FAILED_REFRESHES) (default 6) -g, --grants strings Private discovery IP[:port] or CIDR grants (env: LIVEPEER_DISCOVERY_GRANTS) -p, --print-config Print TOML configuration + --dump Query discovery sources once, print JSON inventory, and exit -h, --help help for livepeer-discovery -v, --version version for livepeer-discovery diff --git a/discovery/app.go b/discovery/app.go index d5d7ecc..bff0483 100644 --- a/discovery/app.go +++ b/discovery/app.go @@ -11,6 +11,7 @@ import ( "net/http" "net/netip" "net/url" + "os" "os/signal" "sync" "syscall" @@ -35,6 +36,7 @@ type Params struct { MaxFailedRefreshes int `default:"6" min:"0" descr:"Consecutive failed refreshes before eviction"` Grants []string `optional:"true" descr:"Private discovery IP[:port] or CIDR grants"` PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print TOML configuration"` + Dump bool `optional:"true" boa:"noconfig,noenv" descr:"Query discovery sources once, print JSON inventory, and exit"` } func Root(out, errOut io.Writer) *cobra.Command { @@ -44,6 +46,9 @@ func Root(out, errOut io.Writer) *cobra.Command { Params: params, RejectUnknown: true, Args: cobra.NoArgs, RunFuncE: func(p *Params, cmd *cobra.Command, _ []string) error { + if p.Dump && p.PrintConfig { + return boa.NewUserInputErrorf("--dump and --print-config are mutually exclusive") + } if p.PrintConfig { app, err := NewServer(*p, slog.Default()) if err != nil { @@ -57,6 +62,19 @@ func Root(out, errOut io.Writer) *cobra.Command { "RefreshInterval": &p.RefreshInterval, "MaxFailedRefreshes": &p.MaxFailedRefreshes, }) } + if p.Dump { + ctx, stop := signal.NotifyContext(cmd.Context(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + app, err := NewServer(*p, slog.New(slog.NewTextHandler(cmd.ErrOrStderr(), nil))) + if err != nil { + return boa.NewUserInputError(err) + } + inventoryOut := cmd.OutOrStdout() + if inventoryOut == os.Stdout { + inventoryOut = dumpStdout{ctx} + } + return app.dump(ctx, inventoryOut) + } return Serve(cmd.Context(), *p, cmd.ErrOrStderr()) }, }, ¶ms.ConfigFile).ToCobra() @@ -68,6 +86,34 @@ func Root(out, errOut io.Writer) *cobra.Command { return cmd } +// dumpStdout lets a signal abort a blocked CLI write. The process exits after +// dump returns, so this must only wrap process-owned stdout. +type dumpStdout struct{ ctx context.Context } + +func (w dumpStdout) Write(data []byte) (int, error) { + type result struct { + n int + err error + } + done := make(chan result, 1) + go func() { + brokenPipe := make(chan os.Signal, 1) + signal.Notify(brokenPipe, syscall.SIGPIPE) + defer signal.Stop(brokenPipe) + n, err := os.Stdout.Write(data) + done <- result{n, err} + }() + select { + case <-w.ctx.Done(): + return 0, w.ctx.Err() + case written := <-done: + if err := w.ctx.Err(); err != nil { + return written.n, err + } + return written.n, written.err + } +} + func Serve(parent context.Context, p Params, logOut io.Writer) (result error) { logger := slog.New(slog.NewTextHandler(logOut, nil)) app, err := NewServer(p, logger) diff --git a/discovery/app_test.go b/discovery/app_test.go index b982301..2a7abc7 100644 --- a/discovery/app_test.go +++ b/discovery/app_test.go @@ -3,14 +3,18 @@ package discovery import ( "bytes" "context" + "encoding/json" + "fmt" "io" "net" "net/http" + "net/http/httptest" "net/netip" "net/url" "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -113,11 +117,141 @@ func TestDiscoveryInvalidConfig(t *testing.T) { {"--nodes=ftp://child.example"}, {"--nodes=https://user:secret@child.example"}, {"--nodes=https://child.example?token=secret"}, {"--nodes=https://child.example#fragment"}, {"--runner-config=unused"}, } { - _, err := execute(t, append(args, "--config=", "--print-config")...) - require.Error(t, err, "%v", args) + for _, mode := range []string{"--print-config", "--dump"} { + out, err := execute(t, append(args, "--config=", mode)...) + require.Error(t, err, "%v", args) + require.Empty(t, out) + } } } +func TestDiscoveryDumpResults(t *testing.T) { + for _, tc := range []struct { + name string + sources []string + nodes, runners bool + denied bool + calls, groups int + metadata int + warnings int + failed bool + }{ + {name: "inventory and normalized duplicates", sources: []string{"/deployment", "/deployment/"}, nodes: true, runners: true, calls: 1, groups: 2, metadata: 1}, + {name: "nodes only", sources: []string{"/deployment"}, nodes: true, calls: 1, metadata: 1}, + {name: "runners only", sources: []string{"/deployment"}, runners: true, calls: 1, groups: 2}, + {name: "disabled", sources: []string{"/deployment"}}, + {name: "no sources", nodes: true, runners: true}, + {name: "empty success with metadata", sources: []string{"/empty"}, nodes: true, runners: true, calls: 1, metadata: 1}, + {name: "empty success without metadata", sources: []string{"/empty"}, runners: true, calls: 1}, + {name: "partial failure", sources: []string{"/failed", "/deployment"}, nodes: true, runners: true, calls: 2, groups: 2, metadata: 1, warnings: 1}, + {name: "all sources fail", sources: []string{"/failed"}, nodes: true, runners: true, calls: 1, warnings: 1, failed: true}, + {name: "redirect is not followed", sources: []string{"/redirect"}, nodes: true, runners: true, calls: 1, warnings: 1, failed: true}, + {name: "private source denied", sources: []string{"/deployment"}, nodes: true, runners: true, denied: true, warnings: 1, failed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var calls atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + switch r.URL.Path { + case "/deployment/discovery": + _, _ = io.WriteString(w, childInventory) + case "/empty/discovery": + _, _ = io.WriteString(w, "[]") + case "/failed/discovery": + w.WriteHeader(http.StatusServiceUnavailable) + case "/redirect/discovery": + http.Redirect(w, r, "/deployment/discovery", http.StatusFound) + default: + t.Errorf("unexpected discovery request: %s", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer upstream.Close() + args := []string{"--dump", "--config=", fmt.Sprintf("--advertise-nodes=%t", tc.nodes), fmt.Sprintf("--advertise-runners=%t", tc.runners)} + if !tc.denied { + args = append(args, "--grants", strings.TrimPrefix(upstream.URL, "http://")) + } + for _, source := range tc.sources { + args = append(args, "--nodes", upstream.URL+source) + } + var out, logs bytes.Buffer + cmd := Root(&out, &logs) + cmd.SetArgs(args) + err := cmd.ExecuteContext(t.Context()) + if tc.failed { + require.ErrorContains(t, err, "all discovery sources failed") + } else { + require.NoError(t, err) + } + require.Equal(t, int32(tc.calls), calls.Load()) + require.Equal(t, tc.warnings, strings.Count(logs.String(), "discovery refresh failed")) + var entries []map[string]json.RawMessage + require.NoError(t, json.Unmarshal(out.Bytes(), &entries)) + require.NotNil(t, entries) + require.True(t, strings.HasSuffix(out.String(), "\n")) + require.Equal(t, 1, strings.Count(out.String(), "\n")) + metadata := 0 + if tc.metadata > 0 { + metadata = 1 + var nodes []discoveryNode + require.NoError(t, json.Unmarshal(entries[0]["nodes"], &nodes)) + require.Len(t, nodes, tc.metadata) + require.Equal(t, upstream.URL+strings.TrimRight(tc.sources[len(tc.sources)-1], "/"), nodes[0].URL, "metadata contains the normalized source URL") + require.False(t, nodes[0].QueriedAt.IsZero()) + require.NotContains(t, entries[0], "address") + } + require.Len(t, entries, metadata+tc.groups) + for _, entry := range entries[metadata:] { + require.Len(t, entry, 2) + require.Contains(t, entry, "address") + require.Contains(t, entry, "runners") + } + }) + } +} + +func TestDiscoveryDumpConfiguration(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + paths := make(chan string, 3) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths <- r.URL.Path + _, _ = io.WriteString(w, "[]") + })) + defer upstream.Close() + config := filepath.Join(t.TempDir(), "discovery.toml") + require.NoError(t, os.WriteFile(config, fmt.Appendf(nil, "Nodes = [%q]\nGrants = [%q]\nAdvertiseNodes = false\n", upstream.URL+"/file", strings.TrimPrefix(upstream.URL, "http://")), 0600)) + for _, source := range []string{"file", "env", "cli"} { + if source == "env" { + t.Setenv("LIVEPEER_DISCOVERY_NODES", upstream.URL+"/env") + } + args := []string{"--dump", "--config", config} + if source == "cli" { + args = append(args, "--nodes", upstream.URL+"/cli") + } + out, err := execute(t, args...) + require.NoError(t, err) + require.Equal(t, "[]\n", out) + select { + case path := <-paths: + require.Equal(t, "/"+source+"/discovery", path) + default: + t.Fatal("configured source was not queried") + } + } + + t.Setenv("LIVEPEER_DISCOVERY_DUMP", "true") + out, err := execute(t, "--config=", "--print-config") + require.NoError(t, err, "environment cannot enable dump mode") + require.NotContains(t, out, "Dump") + out, err = execute(t, "--config=", "--dump", "--print-config") + require.ErrorContains(t, err, "mutually exclusive") + require.Empty(t, out) + require.NoError(t, os.WriteFile(config, []byte("Dump = true\n"), 0600)) + out, err = execute(t, "--dump", "--config", config) + require.ErrorContains(t, err, "Dump") + require.Empty(t, out) +} + func execute(t *testing.T, args ...string) (string, error) { t.Helper() var output bytes.Buffer diff --git a/discovery/server.go b/discovery/server.go index 69bea9c..29104a9 100644 --- a/discovery/server.go +++ b/discovery/server.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "log/slog" "net/http" "net/url" @@ -105,17 +106,55 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (s *Server) Close() { s.collector.Close() } func (s *Server) discovery(w http.ResponseWriter, r *http.Request) { - body, err := json.Marshal(s.discoveryResponse()) + body, err := s.discoveryBody(r.Context()) if err != nil { http.Error(w, "cannot encode discovery inventory", http.StatusInternalServerError) return } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) +} + +func (s *Server) discoveryBody(ctx context.Context) ([]byte, error) { + body, err := json.Marshal(s.discoveryResponse()) + if err != nil { + return nil, fmt.Errorf("cannot encode discovery inventory: %w", err) + } body = append(body, '\n') if len(body) > discovery.MaxBody { - s.logger.WarnContext(r.Context(), "discovery response exceeds signer size limit", "response_bytes", len(body), "max_bytes", discovery.MaxBody) + s.logger.WarnContext(ctx, "discovery response exceeds signer size limit", "response_bytes", len(body), "max_bytes", discovery.MaxBody) } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write(body) + return body, nil +} + +// dump collects one complete inventory on a fresh server, then closes it. +func (s *Server) dump(ctx context.Context, out io.Writer) error { + defer s.Close() + collect := len(s.urls) > 0 && (s.advertiseNodes || s.advertiseRunners) + if collect { + s.collector.Refresh(ctx, false) + } + if err := ctx.Err(); err != nil { + return err + } + body, err := s.discoveryBody(ctx) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + n, err := out.Write(body) + if err == nil && n != len(body) { + err = io.ErrShortWrite + } + if err != nil { + return fmt.Errorf("cannot write discovery inventory: %w", err) + } + if collect && !s.collector.Initialized() { + return errors.New("all discovery sources failed") + } + return nil } // Run refreshes immediately and periodically until ctx ends. Run it once and diff --git a/discovery/server_test.go b/discovery/server_test.go index 92fc4ec..77bef07 100644 --- a/discovery/server_test.go +++ b/discovery/server_test.go @@ -1,8 +1,10 @@ package discovery import ( + "bytes" "context" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -209,15 +211,29 @@ func TestDiscoveryFailureThreshold(t *testing.T) { func TestDiscoveryPreservesWholeRecords(t *testing.T) { const record = `{"url":"https://orch.example/apps/runner/session","app":"test","observed_at":"2000-01-01T00:00:00.123+00:00","mode":"persistent","capacity":4,"capacity_used":1,"capacity_available":3,"gpu":{"id":"0","name":"H100","vram_mb":80000},"price_info":{"price":123456789012345678901234567890,"currency":"wei","unit":"seconds"},"extension":{"value":9007199254740993}}` - s := discoveryTestServer(t, "https://child.example") - s.collector.Client.Transport = discoveryRoundTrip(func(*http.Request) (*http.Response, error) { - return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(`[{"address":"https://orch.example","runners":[` + record + `]}]`))}, nil + s := discoveryTestServer(t, "https://old.example", "https://child.example", "https://tie.example") + var calls atomic.Int32 + s.collector.Client.Transport = discoveryRoundTrip(func(r *http.Request) (*http.Response, error) { + calls.Add(1) + owner, runner := "https://orch.example", record + switch r.URL.Host { + case "old.example": + owner = "https://old-owner.example" + runner = strings.ReplaceAll(runner, "2000-01-01", "1999-01-01") + case "tie.example": + runner = strings.ReplaceAll(runner, `"app":"test"`, `"app":"tie"`) + } + body := `[{"address":"` + owner + `","runners":[` + runner + `]}]` + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(body))}, nil }) - s.collector.Refresh(t.Context(), false) + var dumped bytes.Buffer + require.NoError(t, s.dump(t.Context(), &dumped)) + require.Equal(t, int32(3), calls.Load()) for range 2 { w := httptest.NewRecorder() s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/discovery", nil)) require.Equal(t, 200, w.Code) + require.Equal(t, dumped.String(), w.Body.String(), "dump and HTTP share the same snapshot and encoding") var result []map[string]json.RawMessage require.NoError(t, json.Unmarshal(w.Body.Bytes(), &result)) require.Len(t, result, 2) @@ -252,6 +268,123 @@ func TestDiscoveryPrivateSourcesRequireGrants(t *testing.T) { } } +func TestDiscoveryDumpWaitsForTimeout(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := discoveryTestServer(t, "https://fast.example", "https://slow.example") + s.interval = time.Nanosecond + var logs, out bytes.Buffer + s.collector.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + var calls, writes atomic.Int32 + s.collector.Client.Transport = discoveryRoundTrip(func(r *http.Request) (*http.Response, error) { + calls.Add(1) + if r.URL.Host == "slow.example" { + <-r.Context().Done() + return nil, r.Context().Err() + } + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader("[]"))}, nil + }) + done := make(chan error, 1) + go func() { + done <- s.dump(t.Context(), dumpWriter(func(data []byte) (int, error) { + writes.Add(1) + return out.Write(data) + })) + }() + synctest.Wait() + require.Equal(t, int32(2), calls.Load()) + require.Zero(t, writes.Load(), "dump waits for the last request") + time.Sleep(3 * time.Second) + synctest.Wait() + require.NoError(t, <-done) + require.Equal(t, int32(1), writes.Load()) + require.Contains(t, logs.String(), "error_kind=timeout") + var entries []struct{ Nodes []discoveryNode } + require.NoError(t, json.Unmarshal(out.Bytes(), &entries)) + require.Len(t, entries, 1) + require.Len(t, entries[0].Nodes, 1) + require.Equal(t, "https://fast.example", entries[0].Nodes[0].URL) + time.Sleep(time.Hour) + synctest.Wait() + require.Equal(t, int32(2), calls.Load(), "dump never refreshes periodically") + }) +} + +func TestDiscoveryDumpCancellation(t *testing.T) { + for _, alreadyCanceled := range []bool{false, true} { + t.Run(fmt.Sprint(alreadyCanceled), func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + s := discoveryTestServer(t, "https://child.example") + var active atomic.Int32 + s.collector.Client.Transport = discoveryRoundTrip(func(r *http.Request) (*http.Response, error) { + active.Add(1) + defer active.Add(-1) + <-r.Context().Done() + return nil, r.Context().Err() + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if alreadyCanceled { + cancel() + } + var out bytes.Buffer + done := make(chan error, 1) + go func() { done <- s.dump(ctx, &out) }() + synctest.Wait() + if !alreadyCanceled { + require.Equal(t, int32(1), active.Load()) + cancel() + synctest.Wait() + } + require.ErrorIs(t, <-done, context.Canceled) + require.Zero(t, active.Load(), "pending requests are joined before returning") + require.Empty(t, out.String()) + }) + }) + } +} + +type dumpWriter func([]byte) (int, error) + +func (f dumpWriter) Write(data []byte) (int, error) { return f(data) } + +func TestDiscoveryDumpOutputErrors(t *testing.T) { + writeErr := errors.New("output unavailable") + for _, want := range []error{writeErr, io.ErrShortWrite} { + t.Run(want.Error(), func(t *testing.T) { + s := discoveryTestServer(t, "https://child.example") + s.collector.Client.Transport = discoveryRoundTrip(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 503, Body: io.NopCloser(strings.NewReader(""))}, nil + }) + calls := 0 + err := s.dump(t.Context(), dumpWriter(func(data []byte) (int, error) { + calls++ + require.Equal(t, "[]\n", string(data)) + if want == io.ErrShortWrite { + return 1, nil + } + return 0, writeErr + })) + require.ErrorIs(t, err, want, "output errors take precedence over total source failure") + require.Equal(t, 1, calls) + }) + } +} + +func TestDiscoveryDumpOversizedOutput(t *testing.T) { + var logs, out bytes.Buffer + s, err := NewServer(discoveryTestParams(t, "https://a.example", "https://b.example"), slog.New(slog.NewTextHandler(&logs, nil))) + require.NoError(t, err) + defer s.Close() + s.collector.Client.Transport = discoveryRoundTrip(func(r *http.Request) (*http.Response, error) { + body := fmt.Sprintf(`[{"address":"https://orch.example","runners":[{"url":%q,"app":"test","observed_at":"2000-01-01T00:00:00Z","extension":%q}]}]`, r.URL.String(), strings.Repeat("x", discovery.MaxBody*3/5)) + return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(body))}, nil + }) + require.NoError(t, s.dump(t.Context(), &out)) + require.Greater(t, out.Len(), discovery.MaxBody) + require.True(t, json.Valid(out.Bytes())) + require.Contains(t, logs.String(), "discovery response exceeds signer size limit") +} + func mustURL(t *testing.T, raw string) *url.URL { t.Helper() u, err := url.Parse(raw) diff --git a/docs/discovery.md b/docs/discovery.md index 01d3476..d9e3999 100644 --- a/docs/discovery.md +++ b/docs/discovery.md @@ -14,6 +14,12 @@ livepeer discovery --config /etc/livepeer/discovery.toml [shared configuration](configuration.md) for environment variables, paths, and `--print-config`; the default directory is `~/.lpData/offchain/discovery`. +Use `--dump` to query sources once and exit without listeners or refreshes. It writes a +newline-terminated [discovery JSON array](http-api.md#discovery-http) to stdout; diagnostics +go to stderr. Exit 0 includes empty or partial success, and `[]` when no requests are made +(no sources or both advertising switches disabled). Otherwise, it exits 2, possibly with +partial output. `--dump` is CLI-only and conflicts with `--print-config`. + ## Sources and cache Each source is queried at `/discovery`, retaining any deployment path.