pax_global_header00006660000000000000000000000064151645114370014521gustar00rootroot0000000000000052 comment=36a1101d22b9db3fdace7fe80fd09d75b51b0400 golang-github-lestrrat-go-httprc-3.0.5/000077500000000000000000000000001516451143700200205ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/.github/000077500000000000000000000000001516451143700213605ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/.github/dependabot.yml000066400000000000000000000021051516451143700242060ustar00rootroot00000000000000# To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "gomod" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "daily" target-branch: "v3" labels: - "go" - "dependencies" - "dependabot" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" target-branch: "v3" - package-ecosystem: "gomod" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "daily" target-branch: "v2" labels: - "go" - "dependencies" - "dependabot" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "daily" target-branch: "v2" golang-github-lestrrat-go-httprc-3.0.5/.github/workflows/000077500000000000000000000000001516451143700234155ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/.github/workflows/autodoc.yml000066400000000000000000000010601516451143700255730ustar00rootroot00000000000000name: Auto-Doc on: pull_request: branches: - v3 types: - closed workflow_dispatch: {} jobs: autodoc: runs-on: ubuntu-latest name: "Run commands to generate documentation" if: github.event.pull_request.merged == true steps: - name: Checkout repositor uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Process markdown files run: | find . -name '*.md' | xargs perl tools/autodoc.pl env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} golang-github-lestrrat-go-httprc-3.0.5/.github/workflows/ci.yml000066400000000000000000000012411516451143700245310ustar00rootroot00000000000000name: CI on: [push, pull_request] jobs: build: runs-on: ubuntu-latest strategy: matrix: go: [ '1.24', '1.23' ] name: Go ${{ matrix.go }} test steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Check documentation generator run: | find . -name '*.md' | xargs env AUTODOC_DRYRUN=1 perl tools/autodoc.pl - name: Install Go stable version uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version: ${{ matrix.go }} - name: Test run: go test -v -race -timeout=5m golang-github-lestrrat-go-httprc-3.0.5/.github/workflows/lint.yml000066400000000000000000000010261516451143700251050ustar00rootroot00000000000000name: lint on: [push] jobs: golangci: name: lint runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Install Go stable version uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 with: go-version-file: go.mod - uses: golangci/golangci-lint-action@1e7e51e771db61008b38414a730f564565cf7c20 # v9.2.0 - name: Run go vet run: | go vet ./... golang-github-lestrrat-go-httprc-3.0.5/.gitignore000066400000000000000000000004151516451143700220100ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ golang-github-lestrrat-go-httprc-3.0.5/.golangci.yml000066400000000000000000000032501516451143700224040ustar00rootroot00000000000000version: "2" linters: default: all disable: - cyclop - depguard - dupl - errorlint - exhaustive - forbidigo - funcorder - funlen - gochecknoglobals - gochecknoinits - gocognit - gocritic - gocyclo - godot - godox - gosec - gosmopolitan - govet - inamedparam - ireturn - lll - maintidx - makezero - mnd - nakedret - nestif - nlreturn - noinlineerr - nonamedreturns - paralleltest - tagliatelle - testpackage - thelper - varnamelen - wrapcheck - wsl - wsl_v5 settings: govet: disable: - shadow - fieldalignment enable-all: true exclusions: generated: lax presets: - comments - common-false-positives - legacy - std-error-handling rules: - linters: - staticcheck path: /*.go text: 'ST1003: should not use underscores in package names' - linters: - revive path: /*.go text: don't use an underscore in package name - linters: - contextcheck - exhaustruct path: /*.go - linters: - errcheck path: /main.go - linters: - errcheck - errchkjson - forcetypeassert path: /*_test.go - linters: - forbidigo path: /*_example_test.go paths: - third_party$ - builtin$ - examples$ issues: max-issues-per-linter: 0 max-same-issues: 0 formatters: enable: - gofmt - goimports exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ golang-github-lestrrat-go-httprc-3.0.5/Changes000066400000000000000000000030161516451143700213130ustar00rootroot00000000000000Changes ======= v3.0.5 30 Mar 2026 * Fix periodic check deadlock when number of ready resources exceeds outgoing channel buffer, which caused circular wait between controller and worker goroutines (#113, #116) * Fix proxysink self-deadlock caused by missing mutex unlock on context cancellation path v3.0.4 08 Feb 2026 * Fix worker goroutine dying on sync refresh failure, which could cause deadlocks after repeated failures (lestrrat-go/jwx#1551) * Move ErrNotReady example functions out of client_example_test.go into separate files to avoid triggering autodoc workflow v3.0.3 23 Dec 2025 * Add ErrNotReady error state to avoid waiting for unstable URLs v3.0.2 05 Dev 2025 * Code changes mainly due to upgraded linter. * github.com/lestrrat-go/option upgraded to v2 v3.0.1 18 Aug 2025 * Refresh() no longer requires the resource to be ready. v3.0.0 5 Jun 2025 [Breaking Changes] * The entire API has been re-imagined for Go versions that allow typed parameters v2.0.0 19 Feb 2024 [Breaking Changes] * `Fetcher` type is no longer available. You probably want to provide a customg HTTP client instead via httprc.WithHTTPClient()). * v1.0.4 19 Jul 2022 * Fix sloppy API breakage v1.0.3 19 Jul 2022 * Fix queue insertion in the middle of the queue (#7) v1.0.2 13 Jun 2022 * Properly release a lock when the fetch fails (#5) v1.0.1 29 Mar 2022 * Bump dependency for github.com/lestrrat-go/httpcc to v1.0.1 v1.0.0 29 Mar 2022 * Initial release, refactored out of github.com/lestrrat-go/jwx golang-github-lestrrat-go-httprc-3.0.5/LICENSE000066400000000000000000000020511516451143700210230ustar00rootroot00000000000000MIT License Copyright (c) 2022 lestrrat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang-github-lestrrat-go-httprc-3.0.5/README.md000066400000000000000000000154751516451143700213130ustar00rootroot00000000000000# github.com/lestrrat-go/httprc/v3 ![](https://github.com/lestrrat-go/httprc/v3/workflows/CI/badge.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/lestrrat-go/httprc/v3.svg)](https://pkg.go.dev/github.com/lestrrat-go/httprc/v3) `httprc` is a HTTP "Refresh" Cache. Its aim is to cache a remote resource that can be fetched via HTTP, but keep the cached content up-to-date based on periodic refreshing. # Client A `httprc.Client` object is comprised of 3 parts: The user-facing controller API, the main controller loop, and set of workers that perform the actual fetching. The user-facing controller API is the object returned when you call `(httprc.Client).Start`. ```go ctrl, _ := client.Start(ctx) ``` # Controller API The controller API gives you access to the controller backend that runs asynchronously. All methods take a `context.Context` object because they potentially block. You should be careful to use `context.WithTimeout` to properly set a timeout if you cannot tolerate a blocking operation. # Main Controller Loop The main controller loop is run asynchronously to the controller API. It is single threaded, and it has two reponsibilities. The first is to receive commands from the controller API, and appropriately modify the state of the goroutine, i.e. modify the list of resources it is watching, performing forced refreshes, etc. The other is to periodically wake up and go through the list of resources and re-fetch ones that are past their TTL (in reality, each resource carry a "next-check" time, not a TTL). The main controller loop itself does nothing more: it just kicks these checks periodically. The interval between fetches is changed dynamically based on either the metadata carried with the HTTP responses, such as `Cache-Control` and `Expires` headers, or a constant interval set by the user for a given resource. Between these values, the main controller loop will pick the shortest interval (but no less than 1 second) and checks if resources need updating based on that value. For example, if a resource A has an expiry of 10 minutes and if resource has an expiry of 5 minutes, the main controller loop will attempt to wake up roughly every 5 minutes to check on the resources. When the controller loop detects that a resource needs to be checked for freshness, it will send the resource to the worker pool to be synced. # Interval calculation After the resource is synced, the next fetch is scheduled. The interval to the next fetch is calculated either by using constant intervals, or by heuristics using values from the `http.Response` object. If the constant interval is specified, no extra calculation is performed. If you specify a constant interval of 15 minutes, the resource will be checked every 15 minutes. This is predictable and reliable, but not necessarily efficient. If you do not specify a constant interval, the HTTP response is analyzed for values in `Cache-Control` and `Expires` headers. These values will be compared against a maximum and minimum interval values, which default to 30 days and 15 minutes, respectively. If the values obtained from the headers fall within that range, the value from the header is used. If the value is larger than the maximum, the maximum is used. If the value is lower than the minimum, the minimum is used. # SYNOPSIS ```go package httprc_test import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "time" "github.com/lestrrat-go/httprc/v3" ) func ExampleClient() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() type HelloWorld struct { Hello string `json:"hello"` } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"hello": "world"}) })) options := []httprc.NewClientOption{ // By default the client will allow all URLs (which is what the option // below is explicitly specifying). If you want to restrict what URLs // are allowed, you can specify another whitelist. // // httprc.WithWhitelist(httprc.NewInsecureWhitelist()), } // If you would like to handle errors from asynchronous workers, you can specify a error sink. // This is disabled in this example because the trace logs are dynamic // and thus would interfere with the runnable example test. // options = append(options, httprc.WithErrorSink(errsink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil))))) // If you would like to see the trace logs, you can specify a trace sink. // This is disabled in this example because the trace logs are dynamic // and thus would interfere with the runnable example test. // options = append(options, httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil))))) // Create a new client cl := httprc.NewClient(options...) // Start the client, and obtain a Controller object ctrl, err := cl.Start(ctx) if err != nil { fmt.Println(err.Error()) return } // The following is required if you want to make sure that there are no // dangling goroutines hanging around when you exit. For example, if you // are running tests to check for goroutine leaks, you should call this // function before the end of your test. defer ctrl.Shutdown(time.Second) // Create a new resource that is synchronized every so often // // By default the client will attempt to fetch the resource once // as soon as it can, and then if no other metadata is provided, // it will fetch the resource every 15 minutes. // // If the resource responds with a Cache-Control/Expires header, // the client will attempt to respect that, and will try to fetch // the resource again based on the values obatained from the headers. r, err := httprc.NewResource[HelloWorld](srv.URL, httprc.JSONTransformer[HelloWorld]()) if err != nil { fmt.Println(err.Error()) return } // Add the resource to the controller, so that it starts fetching. // By default, a call to `Add()` will block until the first fetch // succeeds, via an implicit call to `r.Ready()` // You can change this behavior if you specify the `WithWaitReady(false)` // option. ctrl.Add(ctx, r) // if you specified `httprc.WithWaitReady(false)` option, the fetch will happen // "soon", but you're not guaranteed that it will happen before the next // call to `Lookup()`. If you want to make sure that the resource is ready, // you can call `Ready()` like so: /* { tctx, tcancel := context.WithTimeout(ctx, time.Second) defer tcancel() if err := r.Ready(tctx); err != nil { fmt.Println(err.Error()) return } } */ m := r.Resource() fmt.Println(m.Hello) // OUTPUT: // world } ``` source: [client_example_test.go](https://github.com/lestrrat-go/httprc/blob/refs/heads/v3/client_example_test.go) golang-github-lestrrat-go-httprc-3.0.5/backend.go000066400000000000000000000220111516451143700217320ustar00rootroot00000000000000package httprc import ( "context" "fmt" "sync" "time" ) func (c *ctrlBackend) adjustInterval(ctx context.Context, req adjustIntervalRequest) { interval := roundupToSeconds(time.Until(req.resource.Next())) c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got adjust request (current tick interval=%s, next for %q=%s)", c.tickInterval, req.resource.URL(), interval)) if interval < time.Second { interval = time.Second } if c.tickInterval < interval { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: no adjusting required (time to next check %s > current tick interval %s)", interval, c.tickInterval)) } else { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: adjusting tick interval to %s", interval)) c.tickInterval = interval c.check.Reset(interval) } } func (c *ctrlBackend) addResource(ctx context.Context, req addRequest) { r := req.resource if _, ok := c.items[r.URL()]; ok { // Already exists sendReply(ctx, req.reply, struct{}{}, errResourceAlreadyExists) return } c.items[r.URL()] = r if r.MaxInterval() == 0 { r.SetMaxInterval(c.defaultMaxInterval) } if r.MinInterval() == 0 { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: set minimum interval to %s", c.defaultMinInterval)) r.SetMinInterval(c.defaultMinInterval) } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: added resource %q", r.URL())) sendReply(ctx, req.reply, struct{}{}, nil) c.SetTickInterval(time.Nanosecond) } func (c *ctrlBackend) rmResource(ctx context.Context, req rmRequest) { u := req.u if _, ok := c.items[u]; !ok { sendReply(ctx, req.reply, struct{}{}, errResourceNotFound) return } delete(c.items, u) minInterval := oneDay for _, item := range c.items { if d := item.MinInterval(); d < minInterval { minInterval = d } } close(req.reply) c.check.Reset(minInterval) } func (c *ctrlBackend) refreshResource(ctx context.Context, req refreshRequest) { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] START %q", req.u)) defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] END %q", req.u)) u := req.u r, ok := c.items[u] if !ok { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] %s is not registered", req.u)) sendReply(ctx, req.reply, struct{}{}, errResourceNotFound) return } // Note: We don't wait for r.Ready() here because refresh should work // regardless of whether the resource has been fetched before. This allows // refresh to work with resources registered using WithWaitReady(false). r.SetNext(time.Unix(0, 0)) sendWorkerSynchronous(ctx, c.syncoutgoing, synchronousRequest{ resource: r, reply: req.reply, }) c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] sync request for %s sent to worker pool", req.u)) } func (c *ctrlBackend) lookupResource(ctx context.Context, req lookupRequest) { u := req.u r, ok := c.items[u] if !ok { sendReply(ctx, req.reply, nil, errResourceNotFound) return } sendReply(ctx, req.reply, r, nil) } func (c *ctrlBackend) handleRequest(ctx context.Context, req any) { switch req := req.(type) { case adjustIntervalRequest: c.adjustInterval(ctx, req) case addRequest: c.addResource(ctx, req) case rmRequest: c.rmResource(ctx, req) case refreshRequest: c.refreshResource(ctx, req) case lookupRequest: c.lookupResource(ctx, req) default: c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: unknown request type %T", req)) } } func sendWorkerSynchronous(ctx context.Context, ch chan synchronousRequest, r synchronousRequest) { r.resource.SetBusy(true) select { case <-ctx.Done(): case ch <- r: } } func sendReply[T any](ctx context.Context, ch chan backendResponse[T], v T, err error) { defer close(ch) select { case <-ctx.Done(): case ch <- backendResponse[T]{payload: v, err: err}: } } type ctrlBackend struct { items map[string]Resource outgoing chan Resource syncoutgoing chan synchronousRequest incoming chan any // incoming requests to the controller traceSink TraceSink tickInterval time.Duration check *time.Ticker defaultMaxInterval time.Duration defaultMinInterval time.Duration } func (c *ctrlBackend) loop(ctx context.Context, readywg, donewg *sync.WaitGroup) { c.traceSink.Put(ctx, "httprc controller: starting main controller loop") readywg.Done() defer c.traceSink.Put(ctx, "httprc controller: stopping main controller loop") defer donewg.Done() var pending []Resource for { if len(pending) > 0 { // Dispatch pending items while remaining responsive to incoming // requests. This prevents a deadlock where periodicCheck blocks // on c.outgoing while a worker blocks on c.incoming (issue #113). // Skip resources that were removed (or replaced) after periodicCheck // queued them. Without this check, a stale resource could be sent to // a worker, causing an unnecessary fetch and a subsequent // adjustIntervalRequest for a resource that is no longer registered. r := pending[0] // Compare interface values directly. This is safe because all // Resource implementations are pointer types (*ResourceBase[T]), // so the comparison is a pointer identity check. if cur, ok := c.items[r.URL()]; !ok || cur != r { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: skipping pending resource %q (no longer registered or replaced)", r.URL())) r.SetBusy(false) pending = pending[1:] continue } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: dispatching pending resource %q to worker pool (%d remaining)", pending[0].URL(), len(pending))) select { case req := <-c.incoming: c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got request %T (while dispatching)", req)) c.handleRequest(ctx, req) case c.outgoing <- pending[0]: pending = pending[1:] case t := <-c.check.C: pending = append(pending, c.periodicCheck(ctx, t)...) case <-ctx.Done(): return } } else { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for request or tick (tick interval=%s)", c.tickInterval)) select { case req := <-c.incoming: c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got request %T", req)) c.handleRequest(ctx, req) case t := <-c.check.C: pending = c.periodicCheck(ctx, t) case <-ctx.Done(): return } } } } // periodicCheck examines all registered resources and returns those that are // due for refresh. Items are marked busy here so they won't be selected again // on the next tick. The caller (loop) is responsible for dispatching them to // the worker pool, interleaved with incoming request handling, to avoid the // deadlock described in https://github.com/lestrrat-go/httprc/issues/113. func (c *ctrlBackend) periodicCheck(ctx context.Context, t time.Time) []Resource { c.traceSink.Put(ctx, "httprc controller: START periodic check") defer c.traceSink.Put(ctx, "httprc controller: END periodic check") var minNext time.Time minInterval := -1 * time.Second var toDispatch []Resource for _, item := range c.items { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: checking resource %q", item.URL())) next := item.Next() if minNext.IsZero() || next.Before(minNext) { minNext = next } if interval := item.MinInterval(); minInterval < 0 || interval < minInterval { minInterval = interval } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q isBusy=%t, next(%s).After(%s)=%t", item.URL(), item.IsBusy(), next, t, next.After(t))) if item.IsBusy() || next.After(t) { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is busy or not ready yet, skipping", item.URL())) continue } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is ready, queuing for dispatch", item.URL())) item.SetBusy(true) toDispatch = append(toDispatch, item) } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: queued %d resources for dispatch", len(toDispatch))) // Next check is always at the earliest next check + 1 second. // The extra second makes sure that we are _past_ the actual next check time // so we can send the resource to the worker pool if interval := time.Until(minNext); interval > 0 { c.SetTickInterval(roundupToSeconds(interval) + time.Second) c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s", c.tickInterval)) } else { // if we got here, either we have no resources, or all resources are busy. // In this state, it's possible that the interval is less than 1 second, // because we previously set it to a small value for an immediate refresh. // in this case, we want to reset it to a sane value if c.tickInterval < time.Second { c.SetTickInterval(minInterval) c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s after forced refresh", c.tickInterval)) } } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: next check in %s", c.tickInterval)) return toDispatch } func (c *ctrlBackend) SetTickInterval(d time.Duration) { // TODO synchronize if d <= 0 { d = time.Second // ensure positive interval } c.tickInterval = d c.check.Reset(d) } golang-github-lestrrat-go-httprc-3.0.5/client.go000066400000000000000000000114511516451143700216270ustar00rootroot00000000000000package httprc import ( "context" "net/http" "sync" "time" "github.com/lestrrat-go/httprc/v3/errsink" "github.com/lestrrat-go/httprc/v3/proxysink" "github.com/lestrrat-go/httprc/v3/tracesink" ) // setupSink creates and starts a proxy for the given sink if it's not a Nop sink // Returns the sink to use and a cancel function that should be chained with the original cancel func setupSink[T any, S proxysink.Backend[T], NopType any](ctx context.Context, sink S, wg *sync.WaitGroup) (S, context.CancelFunc) { if _, ok := any(sink).(NopType); ok { return sink, func() {} } proxy := proxysink.New[T](sink) wg.Add(1) go func(ctx context.Context, wg *sync.WaitGroup, proxy *proxysink.Proxy[T]) { defer wg.Done() proxy.Run(ctx) }(ctx, wg, proxy) // proxy can be converted to one of the sink subtypes s, ok := any(proxy).(S) if !ok { panic("type assertion failed: proxy cannot be converted to type S") } return s, proxy.Close } // Client is the main entry point for the httprc package. type Client struct { mu sync.Mutex httpcl HTTPClient numWorkers int running bool errSink ErrorSink traceSink TraceSink wl Whitelist defaultMaxInterval time.Duration defaultMinInterval time.Duration } // NewClient creates a new `httprc.Client` object. // // By default ALL urls are allowed. This may not be suitable for you if // are using this in a production environment. You are encouraged to specify // a whitelist using the `WithWhitelist` option. // // NOTE: In future versions, this function signature should be changed to // return an error to properly handle option parsing failures. func NewClient(options ...NewClientOption) *Client { //nolint:staticcheck var errSink ErrorSink = errsink.NewNop() //nolint:staticcheck var traceSink TraceSink = tracesink.NewNop() var wl Whitelist = InsecureWhitelist{} var httpcl HTTPClient = http.DefaultClient defaultMinInterval := DefaultMinInterval defaultMaxInterval := DefaultMaxInterval numWorkers := DefaultWorkers for _, option := range options { switch option.Ident() { case identHTTPClient{}: _ = option.Value(&httpcl) case identWorkers{}: _ = option.Value(&numWorkers) case identErrorSink{}: _ = option.Value(&errSink) case identTraceSink{}: _ = option.Value(&traceSink) case identWhitelist{}: _ = option.Value(&wl) } } if numWorkers <= 0 { numWorkers = 1 } return &Client{ httpcl: httpcl, numWorkers: numWorkers, errSink: errSink, traceSink: traceSink, wl: wl, defaultMinInterval: defaultMinInterval, defaultMaxInterval: defaultMaxInterval, } } // Start sets the client into motion. It will start a number of worker goroutines, // and return a Controller object that you can use to control the execution of // the client. // // If you attempt to call Start more than once, it will return an error. func (c *Client) Start(octx context.Context) (Controller, error) { c.mu.Lock() if c.running { c.mu.Unlock() return nil, errAlreadyRunning } c.running = true c.mu.Unlock() // DON'T CANCEL THIS IN THIS METHOD! It's the responsibility of the // controller to cancel this context. ctx, cancel := context.WithCancel(octx) var donewg sync.WaitGroup // start proxy goroutines that will accept sink requests // and forward them to the appropriate sink errSink, errCancel := setupSink[error, ErrorSink, errsink.Nop](ctx, c.errSink, &donewg) traceSink, traceCancel := setupSink[string, TraceSink, tracesink.Nop](ctx, c.traceSink, &donewg) // Chain the cancel functions ocancel := cancel cancel = func() { ocancel() errCancel() traceCancel() } chbuf := c.numWorkers + 1 incoming := make(chan any, chbuf) outgoing := make(chan Resource, chbuf) syncoutgoing := make(chan synchronousRequest, chbuf) var readywg sync.WaitGroup readywg.Add(c.numWorkers) donewg.Add(c.numWorkers) for range c.numWorkers { wrk := worker{ incoming: incoming, next: outgoing, nextsync: syncoutgoing, errSink: errSink, traceSink: traceSink, httpcl: c.httpcl, } go wrk.Run(ctx, &readywg, &donewg) } tickInterval := oneDay ctrl := &controller{ cancel: cancel, incoming: incoming, shutdown: make(chan struct{}), traceSink: traceSink, wl: c.wl, } backend := &ctrlBackend{ items: make(map[string]Resource), outgoing: outgoing, syncoutgoing: syncoutgoing, incoming: incoming, traceSink: traceSink, tickInterval: tickInterval, check: time.NewTicker(tickInterval), defaultMinInterval: c.defaultMinInterval, defaultMaxInterval: c.defaultMaxInterval, } donewg.Add(1) readywg.Add(1) go backend.loop(ctx, &readywg, &donewg) go func(wg *sync.WaitGroup, ch chan struct{}) { wg.Wait() close(ch) }(&donewg, ctrl.shutdown) readywg.Wait() return ctrl, nil } golang-github-lestrrat-go-httprc-3.0.5/client_example_test.go000066400000000000000000000064141516451143700244040ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "time" "github.com/lestrrat-go/httprc/v3" ) func ExampleClient() { ctx, cancel := context.WithCancel(context.Background()) defer cancel() type HelloWorld struct { Hello string `json:"hello"` } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"hello": "world"}) })) options := []httprc.NewClientOption{ // By default the client will allow all URLs (which is what the option // below is explicitly specifying). If you want to restrict what URLs // are allowed, you can specify another whitelist. // // httprc.WithWhitelist(httprc.NewInsecureWhitelist()), } // If you would like to handle errors from asynchronous workers, you can specify a error sink. // This is disabled in this example because the trace logs are dynamic // and thus would interfere with the runnable example test. // options = append(options, httprc.WithErrorSink(errsink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil))))) // If you would like to see the trace logs, you can specify a trace sink. // This is disabled in this example because the trace logs are dynamic // and thus would interfere with the runnable example test. // options = append(options, httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil))))) // Create a new client cl := httprc.NewClient(options...) // Start the client, and obtain a Controller object ctrl, err := cl.Start(ctx) if err != nil { fmt.Println(err.Error()) return } // The following is required if you want to make sure that there are no // dangling goroutines hanging around when you exit. For example, if you // are running tests to check for goroutine leaks, you should call this // function before the end of your test. defer ctrl.Shutdown(time.Second) // Create a new resource that is synchronized every so often // // By default the client will attempt to fetch the resource once // as soon as it can, and then if no other metadata is provided, // it will fetch the resource every 15 minutes. // // If the resource responds with a Cache-Control/Expires header, // the client will attempt to respect that, and will try to fetch // the resource again based on the values obatained from the headers. r, err := httprc.NewResource[HelloWorld](srv.URL, httprc.JSONTransformer[HelloWorld]()) if err != nil { fmt.Println(err.Error()) return } // Add the resource to the controller, so that it starts fetching. // By default, a call to `Add()` will block until the first fetch // succeeds, via an implicit call to `r.Ready()` // You can change this behavior if you specify the `WithWaitReady(false)` // option. ctrl.Add(ctx, r) // if you specified `httprc.WithWaitReady(false)` option, the fetch will happen // "soon", but you're not guaranteed that it will happen before the next // call to `Lookup()`. If you want to make sure that the resource is ready, // you can call `Ready()` like so: /* { tctx, tcancel := context.WithTimeout(ctx, time.Second) defer tcancel() if err := r.Ready(tctx); err != nil { fmt.Println(err.Error()) return } } */ m := r.Resource() fmt.Println(m.Hello) // OUTPUT: // world } golang-github-lestrrat-go-httprc-3.0.5/client_test.go000066400000000000000000000155351516451143700226750ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "net/http" "net/http/httptest" "strconv" "sync" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/lestrrat-go/httprc/v3/errsink" "github.com/lestrrat-go/httprc/v3/tracesink" "github.com/stretchr/testify/require" ) func TestNewClient(t *testing.T) { t.Parallel() t.Run("default client", func(t *testing.T) { t.Parallel() cl := httprc.NewClient() require.NotNil(t, cl) }) t.Run("with custom options", func(t *testing.T) { t.Parallel() // Test with custom worker count cl := httprc.NewClient(httprc.WithWorkers(10)) require.NotNil(t, cl) // Test with custom HTTP client customHTTPClient := &http.Client{Timeout: 5 * time.Second} cl = httprc.NewClient(httprc.WithHTTPClient(customHTTPClient)) require.NotNil(t, cl) // Test with custom error sink cl = httprc.NewClient(httprc.WithErrorSink(errsink.NewNop())) require.NotNil(t, cl) // Test with custom trace sink cl = httprc.NewClient(httprc.WithTraceSink(tracesink.NewNop())) require.NotNil(t, cl) // Test with whitelist cl = httprc.NewClient(httprc.WithWhitelist(httprc.NewInsecureWhitelist())) require.NotNil(t, cl) }) t.Run("with zero workers", func(t *testing.T) { // Should default to 1 worker when 0 is specified cl := httprc.NewClient(httprc.WithWorkers(0)) require.NotNil(t, cl) }) t.Run("with negative workers", func(t *testing.T) { // Should default to 1 worker when negative is specified cl := httprc.NewClient(httprc.WithWorkers(-1)) require.NotNil(t, cl) }) } func TestClientStart(t *testing.T) { t.Parallel() t.Run("successful start", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) require.NotNil(t, ctrl) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) }) t.Run("start twice should fail", func(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() cl := httprc.NewClient() ctrl1, err := cl.Start(ctx) require.NoError(t, err) require.NotNil(t, ctrl1) defer ctrl1.Shutdown(time.Second) // Second start should fail ctrl2, err := cl.Start(ctx) require.Error(t, err) require.Nil(t, ctrl2) }) t.Run("start with canceled context", func(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) // Start should succeed even with canceled context require.NotNil(t, ctrl) ctrl.Shutdown(time.Second) }) } func TestClientConcurrentStart(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() cl := httprc.NewClient() const numGoroutines = 10 var wg sync.WaitGroup var mu sync.Mutex var successCount, errorCount int var successCtrl httprc.Controller for range numGoroutines { wg.Add(1) go func() { defer wg.Done() ctrl, err := cl.Start(ctx) mu.Lock() defer mu.Unlock() if err != nil { errorCount++ } else { successCount++ if successCtrl == nil { successCtrl = ctrl } else { // If we somehow got multiple successes, clean up ctrl.Shutdown(time.Second) } } }() } wg.Wait() // Exactly one should succeed, others should fail require.Equal(t, 1, successCount, "exactly one start should succeed") require.Equal(t, numGoroutines-1, errorCount, "all other starts should fail") require.NotNil(t, successCtrl, "should have one successful controller") successCtrl.Shutdown(time.Second) } func TestClientWithCustomSinks(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Create mock sinks to capture messages var errorMessages []string var traceMessages []string var mu sync.Mutex errorSink := errsink.NewFunc(func(_ context.Context, err error) { mu.Lock() defer mu.Unlock() errorMessages = append(errorMessages, err.Error()) }) traceSink := tracesink.Func(func(_ context.Context, msg string) { mu.Lock() defer mu.Unlock() traceMessages = append(traceMessages, msg) }) cl := httprc.NewClient( httprc.WithErrorSink(errorSink), httprc.WithTraceSink(traceSink), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Add a resource to generate some trace messages srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"test": "data"}) })) defer srv.Close() resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "custom sinks test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding custom sinks test resource should succeed") // Wait a bit for traces to be generated time.Sleep(100 * time.Millisecond) mu.Lock() defer mu.Unlock() // Should have some trace messages require.NotEmpty(t, traceMessages, "should have received trace messages") // Error messages might be empty if no errors occurred, which is fine // but we test that the sink was properly set up } func TestClientMultipleResources(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Create multiple test servers srv1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"server": "1"}) })) defer srv1.Close() srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"server": "2"}) })) defer srv2.Close() srv3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"server": "3"}) })) defer srv3.Close() cl := httprc.NewClient(httprc.WithWorkers(3)) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Create multiple resources resources := make([]httprc.Resource, 0, 3) servers := []string{srv1.URL, srv2.URL, srv3.URL} for i, serverURL := range servers { resource, err := httprc.NewResource[map[string]string]( serverURL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "creating resource %d", i) resources = append(resources, resource) require.NoError(t, ctrl.Add(ctx, resource), "adding resource %d", i) } // Verify all resources are working for i, resource := range resources { require.NoError(t, resource.Ready(ctx), "resource %d should be ready", i) var data map[string]string require.NoError(t, resource.Get(&data), "getting data from resource %d", i) require.Equal(t, strconv.Itoa(i+1), data["server"], "resource %d should return correct server ID", i) } } golang-github-lestrrat-go-httprc-3.0.5/controller.go000066400000000000000000000145301516451143700225350ustar00rootroot00000000000000package httprc import ( "context" "fmt" "time" ) type Controller interface { // Add adds a new `http.Resource` to the controller. If the resource already exists, // it will return an error. Add(context.Context, Resource, ...AddOption) error // Lookup a `httprc.Resource` by its URL. If the resource does not exist, it // will return an error. Lookup(context.Context, string) (Resource, error) // Remove a `httprc.Resource` from the controller by its URL. If the resource does // not exist, it will return an error. Remove(context.Context, string) error // Refresh forces a resource to be refreshed immediately. If the resource does // not exist, or if the refresh fails, it will return an error. Refresh(context.Context, string) error ShutdownContext(context.Context) error Shutdown(time.Duration) error } type controller struct { cancel context.CancelFunc incoming chan any // incoming requests to the controller shutdown chan struct{} traceSink TraceSink wl Whitelist } // Shutdown is a convenience function that calls ShutdownContext with a // context that has a timeout of `timeout`. func (c *controller) Shutdown(timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() return c.ShutdownContext(ctx) } // ShutdownContext stops the client and all associated goroutines, and waits for them // to finish. If the context is canceled, the function will return immediately: // there fore you should not use the context you used to start the client (because // presumably it's already canceled). // // Waiting for the client shutdown will also ensure that all sinks are properly // flushed. func (c *controller) ShutdownContext(ctx context.Context) error { c.cancel() select { case <-ctx.Done(): return ctx.Err() case <-c.shutdown: return nil } } type ctrlRequest[T any] struct { reply chan T resource Resource u string } type addRequest ctrlRequest[backendResponse[struct{}]] type rmRequest ctrlRequest[backendResponse[struct{}]] type refreshRequest ctrlRequest[backendResponse[struct{}]] type lookupRequest ctrlRequest[backendResponse[Resource]] type synchronousRequest ctrlRequest[backendResponse[struct{}]] type adjustIntervalRequest struct { resource Resource } type backendResponse[T any] struct { payload T err error } func sendBackend[TReq any, TB any](ctx context.Context, backendCh chan any, v TReq, replyCh chan backendResponse[TB]) (TB, error) { select { case <-ctx.Done(): case backendCh <- v: } select { case <-ctx.Done(): var zero TB return zero, ctx.Err() case res := <-replyCh: return res.payload, res.err } } // Lookup returns a resource by its URL. If the resource does not exist, it // will return an error. // // Unfortunately, due to the way typed parameters are handled in Go, we can only // return a Resource object (and not a ResourceBase[T] object). This means that // you will either need to use the `Resource.Get()` method or use a type // assertion to obtain a `ResourceBase[T]` to get to the actual object you are // looking for func (c *controller) Lookup(ctx context.Context, u string) (Resource, error) { reply := make(chan backendResponse[Resource], 1) req := lookupRequest{ reply: reply, u: u, } return sendBackend[lookupRequest, Resource](ctx, c.incoming, req, reply) } // Add adds a new resource to the controller. If the resource already // exists, it will return an error. // // By default this function will automatically wait for the resource to be // fetched once (by calling `r.Ready()`). Note that the `r.Ready()` call will NOT // timeout unless you configure your context object with `context.WithTimeout`. // To disable waiting, you can specify the `WithWaitReady(false)` option. func (c *controller) Add(ctx context.Context, r Resource, options ...AddOption) error { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: START Add(%q)", r.URL())) defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: END Add(%q)", r.URL())) waitReady := true for _, option := range options { switch option.Ident() { case identWaitReady{}: if err := option.Value(&waitReady); err != nil { return fmt.Errorf(`httprc.Controller.Add: failed to parse WaitReady option: %w`, err) } } } if !c.wl.IsAllowed(r.URL()) { return fmt.Errorf(`httprc.Controller.AddResource: cannot add %q: %w`, r.URL(), errBlockedByWhitelist) } reply := make(chan backendResponse[struct{}], 1) req := addRequest{ reply: reply, resource: r, } c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: sending add request for %q to backend", r.URL())) // Send to backend and wait for registration confirmation. // If this succeeds, the resource is in the backend. if _, err := sendBackend[addRequest, struct{}](ctx, c.incoming, req, reply); err != nil { return err } // IMPORTANT: At this point, the resource has been successfully registered // in the backend (stored in c.items map). The backend worker will fetch // this resource periodically. if waitReady { c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for resource %q to be ready", r.URL())) if err := r.Ready(ctx); err != nil { // CHANGE: Wrap Ready() errors with errNotReady to indicate that // registration succeeded but the first fetch hasn't completed. // Using %w twice creates a multi-error chain (Go 1.20+), allowing // errors.Is() to check both errNotReady and the underlying error. return fmt.Errorf("%w: %w", errNotReady, err) } } return nil } // Remove removes a resource from the controller. If the resource does // not exist, it will return an error. func (c *controller) Remove(ctx context.Context, u string) error { reply := make(chan backendResponse[struct{}], 1) req := rmRequest{ reply: reply, u: u, } if _, err := sendBackend[rmRequest, struct{}](ctx, c.incoming, req, reply); err != nil { return err } return nil } // Refresh forces a resource to be refreshed immediately. If the resource does // not exist, or if the refresh fails, it will return an error. // // This function is synchronous, and will block until the resource has been refreshed. func (c *controller) Refresh(ctx context.Context, u string) error { reply := make(chan backendResponse[struct{}], 1) req := refreshRequest{ reply: reply, u: u, } if _, err := sendBackend[refreshRequest, struct{}](ctx, c.incoming, req, reply); err != nil { return err } return nil } golang-github-lestrrat-go-httprc-3.0.5/controller_test.go000066400000000000000000000261031516451143700235730ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "sync" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/stretchr/testify/require" ) func TestControllerAdd(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("add new resource", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]string]( srv.URL+"/test1", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource), "adding new resource to controller should succeed") // Should be able to lookup the resource found, err := ctrl.Lookup(ctx, srv.URL+"/test1") require.NoError(t, err) require.Equal(t, srv.URL+"/test1", found.URL()) }) t.Run("add duplicate resource should fail", func(t *testing.T) { t.Parallel() resource1, err := httprc.NewResource[map[string]string]( srv.URL+"/test2", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) // First add should succeed require.NoError(t, ctrl.Add(ctx, resource1), "first resource addition should succeed") resource2, err := httprc.NewResource[map[string]string]( srv.URL+"/test2", // Same URL httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) // Second add should fail err = ctrl.Add(ctx, resource2) require.Error(t, err) }) t.Run("add with canceled context", func(t *testing.T) { t.Parallel() canceledCtx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately resource, err := httprc.NewResource[map[string]string]( srv.URL+"/test3", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) err = ctrl.Add(canceledCtx, resource) require.Error(t, err) require.Equal(t, context.Canceled, err) }) } func TestControllerLookup(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]string{"path": r.URL.Path}) })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("lookup existing resource", func(t *testing.T) { t.Parallel() testURL := srv.URL + "/lookup-test" resource, err := httprc.NewResource[map[string]string]( testURL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource), "adding resource for lookup test should succeed") found, err := ctrl.Lookup(ctx, testURL) require.NoError(t, err) require.Equal(t, testURL, found.URL()) }) t.Run("lookup non-existent resource", func(t *testing.T) { t.Parallel() nonExistentURL := srv.URL + "/does-not-exist" _, err := ctrl.Lookup(ctx, nonExistentURL) require.Error(t, err) }) t.Run("lookup with canceled context", func(t *testing.T) { t.Parallel() canceledCtx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately _, err := ctrl.Lookup(canceledCtx, srv.URL+"/any") require.Error(t, err) require.Equal(t, context.Canceled, err) }) } func TestControllerRemove(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("remove existing resource", func(t *testing.T) { t.Parallel() testURL := srv.URL + "/remove-test" resource, err := httprc.NewResource[map[string]string]( testURL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) // Add the resource require.NoError(t, ctrl.Add(ctx, resource), "adding resource for removal test should succeed") // Verify it exists _, err = ctrl.Lookup(ctx, testURL) require.NoError(t, err, "resource should be found after adding") // Remove it require.NoError(t, ctrl.Remove(ctx, testURL), "removing existing resource should succeed") // Verify it's gone _, err = ctrl.Lookup(ctx, testURL) require.Error(t, err) }) t.Run("remove non-existent resource", func(t *testing.T) { t.Parallel() nonExistentURL := srv.URL + "/does-not-exist" err := ctrl.Remove(ctx, nonExistentURL) require.Error(t, err) }) t.Run("remove with canceled context", func(t *testing.T) { t.Parallel() canceledCtx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately err := ctrl.Remove(canceledCtx, srv.URL+"/any") require.Error(t, err) require.Equal(t, context.Canceled, err) }) } func TestControllerRefresh(t *testing.T) { t.Parallel() var requestCount int var mu sync.Mutex srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() requestCount++ count := requestCount mu.Unlock() json.NewEncoder(w).Encode(map[string]int{"count": count}) })) t.Cleanup(srv.Close) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("refresh existing resource", func(t *testing.T) { t.Parallel() testURL := srv.URL + "/refresh-test" resource, err := httprc.NewResource[map[string]int]( testURL, httprc.JSONTransformer[map[string]int](), ) require.NoError(t, err) // Add the resource (this will trigger the first request) require.NoError(t, ctrl.Add(ctx, resource), "adding resource for refresh test should succeed") // Get initial count var data1 map[string]int require.NoError(t, resource.Get(&data1), "getting initial data should succeed") initialCount := data1["count"] // Force refresh require.NoError(t, ctrl.Refresh(ctx, testURL), "refreshing resource should succeed") // Get updated count var data2 map[string]int require.NoError(t, resource.Get(&data2), "getting updated data should succeed") newCount := data2["count"] require.Greater(t, newCount, initialCount, "count should have increased after refresh") }) t.Run("refresh non-existent resource", func(t *testing.T) { t.Parallel() nonExistentURL := srv.URL + "/does-not-exist" err := ctrl.Refresh(ctx, nonExistentURL) require.Error(t, err) }) t.Run("refresh with canceled context", func(t *testing.T) { t.Parallel() canceledCtx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately err := ctrl.Refresh(canceledCtx, srv.URL+"/any") require.Error(t, err) require.Equal(t, context.Canceled, err) }) } func TestControllerShutdown(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(srv.Close) t.Run("shutdown with timeout", func(t *testing.T) { t.Parallel() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) // Add a resource resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource), "adding resource for shutdown test should succeed") // Shutdown should complete within timeout require.NoError(t, ctrl.Shutdown(5*time.Second), "shutdown with timeout should succeed") }) t.Run("shutdown with context", func(t *testing.T) { t.Parallel() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) // Add a resource resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource), "adding resource for shutdown context test should succeed") shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) defer shutdownCancel() require.NoError(t, ctrl.ShutdownContext(shutdownCtx), "shutdown with context should succeed") }) t.Run("shutdown with canceled context", func(t *testing.T) { t.Parallel() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) canceledCtx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately err = ctrl.ShutdownContext(canceledCtx) require.Error(t, err) require.Equal(t, context.Canceled, err) // Clean shutdown with proper context require.NoError(t, ctrl.Shutdown(time.Second), "clean shutdown should succeed") }) } func TestControllerConcurrentOperations(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { json.NewEncoder(w).Encode(map[string]string{"url": r.URL.Path}) })) defer srv.Close() cl := httprc.NewClient(httprc.WithWorkers(5)) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) const numGoroutines = 10 const numOperationsPerGoroutine = 5 var wg sync.WaitGroup var addedURLs sync.Map // Concurrent adds for i := range numGoroutines { wg.Add(1) go func(i int) { defer wg.Done() for j := range numOperationsPerGoroutine { testURL := fmt.Sprintf("%s/concurrent-test-%d-%d", srv.URL, i, j) resource, err := httprc.NewResource[map[string]string]( testURL, httprc.JSONTransformer[map[string]string](), ) if err != nil { t.Errorf("failed to create resource: %v", err) return } err = ctrl.Add(ctx, resource) if err != nil { t.Errorf("failed to add resource %s: %v", testURL, err) return } addedURLs.Store(testURL, true) } }(i) } wg.Wait() // Verify all resources can be looked up addedURLs.Range(func(key, _ any) bool { testURL := key.(string) _, err := ctrl.Lookup(ctx, testURL) require.NoError(t, err, "should be able to lookup %s", testURL) return true }) // Concurrent lookups and refreshes wg = sync.WaitGroup{} for i := range numGoroutines { wg.Add(1) go func(i int) { defer wg.Done() for j := range numOperationsPerGoroutine { testURL := fmt.Sprintf("%s/concurrent-test-%d-%d", srv.URL, i, j) // Lookup _, err := ctrl.Lookup(ctx, testURL) if err != nil { t.Errorf("failed to lookup resource %s: %v", testURL, err) return } // Refresh err = ctrl.Refresh(ctx, testURL) if err != nil { t.Errorf("failed to refresh resource %s: %v", testURL, err) return } } }(i) } wg.Wait() } golang-github-lestrrat-go-httprc-3.0.5/err_not_ready_basic_handling_example_test.go000066400000000000000000000031021516451143700307560ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "time" "github.com/lestrrat-go/httprc/v3" ) // Example_err_not_ready_basic_handling demonstrates basic handling of ErrNotReady func Example_err_not_ready_basic_handling() { ctx := context.Background() // Create a server that's slow to respond srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) if err != nil { fmt.Println("Failed to start client:", err) return } defer ctrl.Shutdown(time.Second) resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) if err != nil { fmt.Println("Failed to create resource:", err) return } // Add with timeout - will return ErrNotReady addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) defer cancel() err = ctrl.Add(addCtx, resource) if err != nil { if errors.Is(err, httprc.ErrNotReady()) { // Resource registered, will fetch in background fmt.Println("Resource registered but not ready yet") fmt.Println("Safe to continue with application startup") return } // Registration failed fmt.Println("Failed to register resource:", err) return } // Resource registered AND ready with data fmt.Println("Resource ready") // OUTPUT: // Resource registered but not ready yet // Safe to continue with application startup } golang-github-lestrrat-go-httprc-3.0.5/err_not_ready_checking_underlying_error_example_test.go000066400000000000000000000033101516451143700332560ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "time" "github.com/lestrrat-go/httprc/v3" ) // Example_err_not_ready_checking_underlying_error demonstrates how to check // the underlying error wrapped by ErrNotReady func Example_err_not_ready_checking_underlying_error() { ctx := context.Background() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) if err != nil { fmt.Println("Failed to start client:", err) return } defer ctrl.Shutdown(time.Second) resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) if err != nil { fmt.Println("Failed to create resource:", err) return } // Add with timeout addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) defer cancel() err = ctrl.Add(addCtx, resource) if err != nil { if errors.Is(err, httprc.ErrNotReady()) { // Resource registered, check why it's not ready // errors.Is() automatically unwraps the error chain if errors.Is(err, context.DeadlineExceeded) { fmt.Println("Resource registered but timed out waiting for data") fmt.Println("Will continue fetching in background") } else { fmt.Printf("Resource registered but not ready: %v\n", err) } return } // Registration failed fmt.Println("Registration failed:", err) return } fmt.Println("Resource ready") // OUTPUT: // Resource registered but timed out waiting for data // Will continue fetching in background } golang-github-lestrrat-go-httprc-3.0.5/err_not_ready_retry_logic_example_test.go000066400000000000000000000036331516451143700303640ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" "time" "github.com/lestrrat-go/httprc/v3" ) // Example_err_not_ready_retry_logic demonstrates proper retry logic that // distinguishes between registration failures and ErrNotReady func Example_err_not_ready_retry_logic() { ctx := context.Background() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) if err != nil { fmt.Println("Failed to start client:", err) return } defer ctrl.Shutdown(time.Second) var resource httprc.Resource url := srv.URL // Retry logic: only retry registration failures for attempt := 1; attempt <= 3; attempt++ { resource, err = httprc.NewResource[map[string]string]( url, httprc.JSONTransformer[map[string]string](), ) if err != nil { fmt.Printf("Attempt %d: failed to create resource: %v\n", attempt, err) continue } addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) err = ctrl.Add(addCtx, resource) cancel() if err == nil { // Success - registered and ready fmt.Println("Resource registered and ready") return } if errors.Is(err, httprc.ErrNotReady()) { // Registered successfully, just not ready yet // Don't retry Add() - it would fail with duplicate URL fmt.Printf("Attempt %d: Resource registered, not ready yet\n", attempt) fmt.Println("Resource will fetch in background, continuing...") return } // Registration failed - retry fmt.Printf("Attempt %d: Registration failed: %v\n", attempt, err) if attempt < 3 { time.Sleep(time.Second * time.Duration(attempt)) } } // OUTPUT: // Attempt 1: Resource registered, not ready yet // Resource will fetch in background, continuing... } golang-github-lestrrat-go-httprc-3.0.5/errors.go000066400000000000000000000064621516451143700216730ustar00rootroot00000000000000package httprc import "errors" var errResourceAlreadyExists = errors.New(`resource already exists`) func ErrResourceAlreadyExists() error { return errResourceAlreadyExists } var errAlreadyRunning = errors.New(`client is already running`) func ErrAlreadyRunning() error { return errAlreadyRunning } var errResourceNotFound = errors.New(`resource not found`) func ErrResourceNotFound() error { return errResourceNotFound } var errTransformerRequired = errors.New(`transformer is required`) func ErrTransformerRequired() error { return errTransformerRequired } var errURLCannotBeEmpty = errors.New(`URL cannot be empty`) func ErrURLCannotBeEmpty() error { return errURLCannotBeEmpty } var errUnexpectedStatusCode = errors.New(`unexpected status code`) func ErrUnexpectedStatusCode() error { return errUnexpectedStatusCode } var errTransformerFailed = errors.New(`failed to transform response body`) func ErrTransformerFailed() error { return errTransformerFailed } var errRecoveredFromPanic = errors.New(`recovered from panic`) func ErrRecoveredFromPanic() error { return errRecoveredFromPanic } var errBlockedByWhitelist = errors.New(`blocked by whitelist`) func ErrBlockedByWhitelist() error { return errBlockedByWhitelist } var errNotReady = errors.New(`resource registered but not ready`) // ErrNotReady returns a sentinel error indicating that the resource was // successfully registered with the backend and is being actively managed, // but the first fetch and transformation has not completed successfully yet. // // This error is returned by Add() when: // - The resource was successfully added to the backend (registration succeeded) // - WithWaitReady(true) was specified (the default) // - The Ready() call failed (timeout, transform error, context cancelled, etc.) // // When Add() returns this error, the resource IS in the backend's resource map // and will continue to be fetched periodically in the background according to // the refresh interval. The application can safely proceed - the resource data // may become available later when a fetch succeeds. // // IMPORTANT: "Not ready" means the first fetch and transformation has not completed // successfully. The resource may eventually become ready (if the transformation // succeeds on a subsequent retry), or it may never become ready (if the data is // permanently invalid or the server is unreachable). The backend will continue // retrying according to the configured refresh interval. // // The underlying error (context deadline, transform failure, etc.) is wrapped // using Go 1.20+ multiple error wrapping and can be examined with errors.Is() // or errors.As(). You do not need to manually unwrap the error. // // Example: // // err := ctrl.Add(ctx, resource) // if err != nil { // if errors.Is(err, httprc.ErrNotReady()) { // // Resource registered, will fetch in background // log.Print("Resource not ready yet, continuing startup") // // // Can also check the underlying cause // if errors.Is(err, context.DeadlineExceeded) { // log.Print("Timed out waiting for first fetch") // } // return nil // } // // Registration failed // return fmt.Errorf("failed to register resource: %w", err) // } // // Resource registered AND ready with data func ErrNotReady() error { return errNotReady } golang-github-lestrrat-go-httprc-3.0.5/errsink/000077500000000000000000000000001516451143700214755ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/errsink/errsink.go000066400000000000000000000024051516451143700235020ustar00rootroot00000000000000package errsink import ( "context" "log/slog" ) type Interface interface { Put(context.Context, error) } // Nop is an ErrorSink that does nothing. It does not require // any initialization, so the zero value can be used. type Nop struct{} // NewNop returns a new NopErrorSink object. The constructor // is provided for consistency. func NewNop() Interface { return Nop{} } // Put for NopErrorSink does nothing. func (Nop) Put(context.Context, error) {} type SlogLogger interface { Log(context.Context, slog.Level, string, ...any) } type slogSink struct { logger SlogLogger } // NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger func NewSlog(l SlogLogger) Interface { return &slogSink{ logger: l, } } func (s *slogSink) Put(ctx context.Context, v error) { s.logger.Log(ctx, slog.LevelError, v.Error()) } // FuncSink is an ErrorSink that calls a function with the error. type FuncSink struct { fn func(context.Context, error) } // NewFunc returns a new FuncSink that calls the provided function with errors. func NewFunc(fn func(context.Context, error)) Interface { return &FuncSink{fn: fn} } // Put calls the function with the error. func (f *FuncSink) Put(ctx context.Context, err error) { if f.fn != nil { f.fn(ctx, err) } } golang-github-lestrrat-go-httprc-3.0.5/errsink/errsink_test.go000066400000000000000000000072531516451143700245470ustar00rootroot00000000000000package errsink_test import ( "context" "errors" "log/slog" "testing" "github.com/lestrrat-go/httprc/v3/errsink" ) func TestNop(t *testing.T) { t.Parallel() tests := []struct { name string err error }{ { name: "nil error", err: nil, }, { name: "simple error", //nolint:err113 err: errors.New("test error"), }, { name: "wrapped error", err: errors.Join(errors.New("base error"), errors.New("wrapped error")), //nolint:err113 }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() sink := errsink.NewNop() // Should not panic or do anything ctx := context.Background() sink.Put(ctx, tt.err) }) } } func TestNopZeroValue(t *testing.T) { t.Parallel() // Test that zero value can be used directly var sink errsink.Nop ctx := context.Background() err := errors.New("test error") //nolint:err113 // Should not panic sink.Put(ctx, err) } type mockSlogger struct { logs []logEntry } type logEntry struct { level slog.Level msg string args []any } func (m *mockSlogger) Log(_ context.Context, level slog.Level, msg string, args ...any) { m.logs = append(m.logs, logEntry{ level: level, msg: msg, args: args, }) } func TestSlogSink(t *testing.T) { t.Parallel() tests := []struct { name string err error wantMsg string wantArgs int }{ { name: "simple error", err: errors.New("test error"), //nolint:err113 wantMsg: "test error", wantArgs: 0, }, { name: "error with formatting", err: errors.New("error with %d number"), //nolint:err113 wantMsg: "error with %d number", wantArgs: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() logger := &mockSlogger{} sink := errsink.NewSlog(logger) ctx := context.Background() sink.Put(ctx, tt.err) if len(logger.logs) != 1 { t.Errorf("expected 1 log entry, got %d", len(logger.logs)) return } entry := logger.logs[0] // Note: We don't store context to avoid containedctx lint issue if entry.level != slog.LevelError { t.Errorf("expected level %v, got %v", slog.LevelError, entry.level) } if entry.msg != tt.wantMsg { t.Errorf("expected message %q, got %q", tt.wantMsg, entry.msg) } if len(entry.args) != tt.wantArgs { t.Errorf("expected %d args, got %d", tt.wantArgs, len(entry.args)) } }) } } func TestSlogSinkWithNilError(t *testing.T) { t.Parallel() logger := &mockSlogger{} sink := errsink.NewSlog(logger) ctx := context.Background() // This should panic because nil error cannot call Error() method defer func() { if r := recover(); r == nil { t.Error("expected panic when putting nil error to slog sink") } }() sink.Put(ctx, nil) } func TestSlogSinkMultipleErrors(t *testing.T) { t.Parallel() logger := &mockSlogger{} sink := errsink.NewSlog(logger) ctx := context.Background() errors := []error{ errors.New("first error"), //nolint:err113 errors.New("second error"), //nolint:err113 errors.New("third error"), //nolint:err113 } for _, err := range errors { sink.Put(ctx, err) } if len(logger.logs) != len(errors) { t.Errorf("expected %d log entries, got %d", len(errors), len(logger.logs)) return } for i, err := range errors { if logger.logs[i].msg != err.Error() { t.Errorf("log entry %d: expected message %q, got %q", i, err.Error(), logger.logs[i].msg) } } } func TestInterface(t *testing.T) { t.Parallel() // Ensure types implement the interface //nolint:staticcheck { var _ errsink.Interface = (*errsink.Nop)(nil) var _ errsink.Interface = errsink.NewNop() var _ errsink.Interface = errsink.NewSlog(&mockSlogger{}) } } golang-github-lestrrat-go-httprc-3.0.5/go.mod000066400000000000000000000006051516451143700211270ustar00rootroot00000000000000module github.com/lestrrat-go/httprc/v3 go 1.23.0 toolchain go1.24.4 require ( github.com/lestrrat-go/blackmagic v1.0.4 github.com/lestrrat-go/httpcc v1.0.1 github.com/lestrrat-go/option/v2 v2.0.0 github.com/stretchr/testify v1.11.1 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) golang-github-lestrrat-go-httprc-3.0.5/go.sum000066400000000000000000000034111516451143700211520ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLOcID3Ss= github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= golang-github-lestrrat-go-httprc-3.0.5/httprc.go000066400000000000000000000051251516451143700216560ustar00rootroot00000000000000package httprc import ( "context" "net/http" "time" "github.com/lestrrat-go/httprc/v3/errsink" "github.com/lestrrat-go/httprc/v3/tracesink" ) // Buffer size constants const ( // ReadBufferSize is the default buffer size for reading HTTP responses (10MB) ReadBufferSize = 1024 * 1024 * 10 // MaxBufferSize is the maximum allowed buffer size (1GB) MaxBufferSize = 1024 * 1024 * 1000 ) // Client worker constants const ( // DefaultWorkers is the default number of worker goroutines DefaultWorkers = 5 ) // Interval constants const ( // DefaultMaxInterval is the default maximum interval between fetches (30 days) DefaultMaxInterval = 24 * time.Hour * 30 // DefaultMinInterval is the default minimum interval between fetches (15 minutes) DefaultMinInterval = 15 * time.Minute // oneDay is used internally for time calculations oneDay = 24 * time.Hour ) // utility to round up intervals to the nearest second func roundupToSeconds(d time.Duration) time.Duration { if diff := d % time.Second; diff > 0 { return d + time.Second - diff } return d } // ErrorSink is an interface that abstracts a sink for errors. type ErrorSink = errsink.Interface type TraceSink = tracesink.Interface // HTTPClient is an interface that abstracts a "net/http".Client, so that // users can provide their own implementation of the HTTP client, if need be. type HTTPClient interface { Do(*http.Request) (*http.Response, error) } // Transformer is used to convert the body of an HTTP response into an appropriate // object of type T. type Transformer[T any] interface { Transform(context.Context, *http.Response) (T, error) } // TransformFunc is a function type that implements the Transformer interface. type TransformFunc[T any] func(context.Context, *http.Response) (T, error) func (f TransformFunc[T]) Transform(ctx context.Context, res *http.Response) (T, error) { return f(ctx, res) } // Resource is a single resource that can be retrieved via HTTP, and (possibly) transformed // into an arbitrary object type. // // Realistically, there is no need for third-parties to implement this interface. This exists // to provide a way to aggregate `httprc.ResourceBase` objects with different specialized types // into a single collection. // // See ResourceBase for details type Resource interface { //nolint:interfacebloat Get(any) error Next() time.Time SetNext(time.Time) URL() string Sync(context.Context) error ConstantInterval() time.Duration MaxInterval() time.Duration SetMaxInterval(time.Duration) MinInterval() time.Duration SetMinInterval(time.Duration) IsBusy() bool SetBusy(bool) Ready(context.Context) error } golang-github-lestrrat-go-httprc-3.0.5/httprc_test.go000066400000000000000000001001361516451143700227130ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "io" "log/slog" "net/http" "net/http/httptest" "os" "strconv" "sync" "sync/atomic" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/lestrrat-go/httprc/v3/tracesink" "github.com/stretchr/testify/require" ) func TestResource(t *testing.T) { const dummy = "https://127.0.0.1:99999999" r, err := httprc.NewResource[[]byte](dummy, httprc.BytesTransformer()) require.NoError(t, err, `NewResource should succeed`) require.Equal(t, httprc.DefaultMinInterval, r.MinInterval(), `r.MinInterval should return DefaultMinInterval`) require.Equal(t, httprc.DefaultMaxInterval, r.MaxInterval(), `r.MaxInterval should return DefaultMaxInterval`) r, err = httprc.NewResource[[]byte](dummy, httprc.BytesTransformer(), httprc.WithMinInterval(12*time.Second)) require.NoError(t, err, `NewResource should succeed`) require.Equal(t, 12*time.Second, r.MinInterval(), `r.MinInterval should return expected value`) require.Equal(t, httprc.DefaultMaxInterval, r.MaxInterval(), `r.MaxInterval should return DefaultMaxInterval`) r, err = httprc.NewResource[[]byte](dummy, httprc.BytesTransformer(), httprc.WithMaxInterval(12*time.Second)) require.NoError(t, err, `NewResource should succeed`) require.Equal(t, httprc.DefaultMinInterval, r.MinInterval(), `r.MinInterval should return DefaultMinInterval`) require.Equal(t, 12*time.Second, r.MaxInterval(), `r.MaxInterval should return expected value`) } func TestClient(t *testing.T) { type Hello struct { Hello string `json:"hello"` } start := time.Now() h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "max-age=2") var version string if time.Since(start) > 2*time.Second { version = "2" } switch r.URL.Path { case "/json/helloptr", "/json/hello", "/json/hellomap": w.Header().Set("Content-Type", "application/json") switch version { case "2": w.Write([]byte(`{"hello":"world2"}`)) default: w.Write([]byte(`{"hello":"world"}`)) } case "/int": w.Header().Set("Content-Type", "text/plain") w.Write([]byte(`42`)) case "/string": w.Header().Set("Content-Type", "text/plain") w.Write([]byte(`Lorem ipsum dolor sit amet`)) case "/custom": } }) srv := httptest.NewServer(h) defer srv.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() options := []httprc.NewClientOption{ // httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))), } cl := httprc.NewClient(options...) ctrl, err := cl.Start(ctx) require.NoError(t, err, `cl.Run should succeed`) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) testcases := []struct { URL string Create func() (httprc.Resource, error) Expected any Expected2 any }{ { URL: srv.URL + "/json/helloptr", Create: func() (httprc.Resource, error) { r, err := httprc.NewResource[*Hello](srv.URL+"/json/helloptr", httprc.JSONTransformer[*Hello]()) if err != nil { return nil, err } r.SetMinInterval(time.Second) return r, nil }, Expected: &Hello{Hello: "world"}, Expected2: &Hello{Hello: "world2"}, }, { URL: srv.URL + "/json/hello", Create: func() (httprc.Resource, error) { r, err := httprc.NewResource[Hello](srv.URL+"/json/hello", httprc.JSONTransformer[Hello]()) if err != nil { return nil, err } r.SetMinInterval(time.Second) return r, nil }, Expected: Hello{Hello: "world"}, Expected2: Hello{Hello: "world2"}, }, { URL: srv.URL + "/json/hellomap", Create: func() (httprc.Resource, error) { r, err := httprc.NewResource[map[string]any](srv.URL+"/json/hellomap", httprc.JSONTransformer[map[string]any]()) if err != nil { return nil, err } r.SetMinInterval(time.Second) return r, nil }, Expected: map[string]any{"hello": "world"}, Expected2: map[string]any{"hello": "world2"}, }, { URL: srv.URL + "/int", Create: func() (httprc.Resource, error) { return httprc.NewResource[int](srv.URL+"/int", httprc.TransformFunc[int](func(_ context.Context, res *http.Response) (int, error) { buf, err := io.ReadAll(res.Body) if err != nil { return 0, err } return strconv.Atoi(string(buf)) })) }, Expected: 42, }, { URL: srv.URL + "/string", Create: func() (httprc.Resource, error) { return httprc.NewResource[string](srv.URL+"/string", httprc.TransformFunc[string](func(_ context.Context, res *http.Response) (string, error) { buf, err := io.ReadAll(res.Body) if err != nil { return "", err } return string(buf), nil })) }, Expected: "Lorem ipsum dolor sit amet", }, } for _, tc := range testcases { t.Run(tc.URL, func(t *testing.T) { r, err := tc.Create() require.NoError(t, err, `NewResource should succeed`) require.NoError(t, ctrl.Add(ctx, r), `ctrl.Add should succeed`) require.NoError(t, r.Ready(ctx), `r.Ready should succeed`) var dst any require.NoError(t, r.Get(&dst), `r.Get should succeed`) require.Equal(t, tc.Expected, dst, `r.Get should return expected value`) }) } time.Sleep(6 * time.Second) for _, tc := range testcases { t.Run("Lookup "+tc.URL, func(t *testing.T) { r, err := ctrl.Lookup(ctx, tc.URL) require.NoError(t, err, `ctrl.Lookup should succeed`) require.Equal(t, tc.URL, r.URL(), `r.URL should return expected value`) var dst any require.NoError(t, r.Get(&dst), `r.Get should succeed`) expected := tc.Expected2 if expected == nil { expected = tc.Expected } require.Equal(t, expected, dst, `r.Resource should return expected value`) }) } } func TestRefresh(t *testing.T) { count := 0 var mu sync.Mutex h := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() defer mu.Unlock() count++ json.NewEncoder(w).Encode(map[string]any{"count": count}) }) srv := httptest.NewServer(h) defer srv.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() options := []httprc.NewClientOption{ httprc.WithWhitelist(httprc.NewInsecureWhitelist()), } cl := httprc.NewClient(options...) ctrl, err := cl.Start(ctx) require.NoError(t, err, `cl.Run should succeed`) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) r, err := httprc.NewResource[map[string]int](srv.URL, httprc.JSONTransformer[map[string]int]()) require.NoError(t, err, `NewResource should succeed`) require.NoError(t, ctrl.Add(ctx, r), `ctrl.Add should succeed`) require.NoError(t, r.Ready(ctx), `r.Ready should succeed`) for i := 1; i <= 5; i++ { m := r.Resource() require.Equal(t, i, m["count"], `r.Resource should return expected value`) require.NoError(t, ctrl.Refresh(ctx, srv.URL), `r.Refresh should succeed`) } } func Test_gh74(t *testing.T) { // Test server that returns simple JSON data testData := `{"test": "data"}` server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(testData)) })) t.Cleanup(func() { server.Close() }) // Create httprc client with trace logging client := httprc.NewClient( httprc.WithTraceSink( tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))), ) // Create a resource that transforms bytes resource, err := httprc.NewResource[[]byte](server.URL, httprc.BytesTransformer()) require.NoError(t, err, "failed to create resource") ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() // Start the client to get a controller ctrl, err := client.Start(ctx) require.NoError(t, err, "failed to start client") defer func() { _ = ctrl.Shutdown(time.Second) }() // Test the original issue: Add with WithWaitReady(false) followed by Refresh calls // This would block before the fix require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "Add should succeed") // These refresh calls would block indefinitely before the fix for i := range 10 { err = ctrl.Refresh(ctx, server.URL) require.NoError(t, err, "refresh should succeed on iteration %d", i) // Verify we can lookup the resource res, err := ctrl.Lookup(ctx, server.URL) require.NoError(t, err, "lookup should succeed") require.NotNil(t, res, "resource should not be nil") } } // TestAdd_returns_err_not_ready_when_ready_fails tests that ErrNotReady is returned // when Ready() fails but registration succeeded func TestAdd_returns_err_not_ready_when_ready_fails(t *testing.T) { t.Parallel() // Setup: Create mock HTTP server that returns invalid JSON server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(`{invalid json`)) })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]any]( server.URL, httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err) // Act: Add with timeout (will fail on Ready due to invalid JSON) addCtx, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() err = ctrl.Add(addCtx, resource) // Assert: Should return ErrNotReady require.Error(t, err, "expected error when Ready() fails") require.ErrorIs(t, err, httprc.ErrNotReady(), "expected ErrNotReady") // Verify resource IS in backend existing, lookupErr := ctrl.Lookup(ctx, server.URL) require.NoError(t, lookupErr, "resource should be in backend after ErrNotReady") require.NotNil(t, existing, "resource should exist in backend") } // TestAdd_does_not_return_err_not_ready_when_registration_fails tests that ErrNotReady // is NOT returned when registration fails (before Ready() is called) func TestAdd_does_not_return_err_not_ready_when_registration_fails(t *testing.T) { t.Parallel() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Add first resource resource1, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource1, httprc.WithWaitReady(false))) // Try to add duplicate resource2, err := httprc.NewResource[map[string]string]( server.URL, // Same URL httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) err = ctrl.Add(ctx, resource2) // Assert: Should NOT be ErrNotReady (registration failed before Ready) require.Error(t, err, "expected error for duplicate URL") require.NotErrorIs(t, err, httprc.ErrNotReady(), "should not return ErrNotReady for registration failure") } // TestAdd_with_wait_ready_false_never_returns_err_not_ready tests that WithWaitReady(false) // never returns ErrNotReady because Ready() is not called func TestAdd_with_wait_ready_false_never_returns_err_not_ready(t *testing.T) { t.Parallel() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(`{invalid json`)) })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]any]( server.URL, httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err) // Act: Add with WithWaitReady(false) err = ctrl.Add(ctx, resource, httprc.WithWaitReady(false)) // Assert: Should succeed (no error) because we didn't wait for Ready require.NoError(t, err, "expected no error with WithWaitReady(false)") // Verify resource is in backend existing, lookupErr := ctrl.Lookup(ctx, server.URL) require.NoError(t, lookupErr, "resource should be in backend") require.NotNil(t, existing, "expected resource in backend") // Verify Ready() will fail separately readyCtx, cancel := context.WithTimeout(ctx, 1*time.Second) defer cancel() readyErr := resource.Ready(readyCtx) require.Error(t, readyErr, "expected Ready() to fail with invalid JSON") } // TestErrNotReady_wraps_underlying_error tests that ErrNotReady wraps the // underlying error using Go 1.20+ multi-error wrapping func TestErrNotReady_wraps_underlying_error(t *testing.T) { t.Parallel() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) // Longer than context timeout w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) // Add with short timeout addCtx, cancel := context.WithTimeout(ctx, 500*time.Millisecond) defer cancel() err = ctrl.Add(addCtx, resource) // Assert: Should be ErrNotReady wrapping DeadlineExceeded require.ErrorIs(t, err, httprc.ErrNotReady(), "expected ErrNotReady") require.ErrorIs(t, err, context.DeadlineExceeded, "expected wrapped DeadlineExceeded") } // TestAdd_whitelist_blocked_not_err_not_ready tests that whitelist blocking // returns a non-ErrNotReady error func TestAdd_whitelist_blocked_not_err_not_ready(t *testing.T) { t.Parallel() ctx := context.Background() cl := httprc.NewClient(httprc.WithWhitelist(httprc.WhitelistFunc(func(_ string) bool { return false // Block everything }))) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]any]( "https://example.com", httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err) err = ctrl.Add(ctx, resource) // Assert: Should return whitelist error, not ErrNotReady require.Error(t, err, "expected whitelist error") require.NotErrorIs(t, err, httprc.ErrNotReady(), "whitelist blocking should not return ErrNotReady") require.ErrorIs(t, err, httprc.ErrBlockedByWhitelist(), "expected ErrBlockedByWhitelist") } // TestAdd_context_cancelled_before_send_backend_not_err_not_ready tests that context // cancellation before registration completes returns non-ErrNotReady func TestAdd_context_cancelled_before_send_backend_not_err_not_ready(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately cl := httprc.NewClient() ctrl, err := cl.Start(context.Background()) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]any]( "https://example.com", httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err) err = ctrl.Add(ctx, resource) // Assert: Should return context error, not ErrNotReady require.Error(t, err, "expected context error") require.NotErrorIs(t, err, httprc.ErrNotReady(), "context cancellation should not return ErrNotReady") require.ErrorIs(t, err, context.Canceled, "expected context.Canceled") } // TestAdd_retry_logic_distinguishes_errors tests that retry logic can properly // distinguish between registration failures and ErrNotReady func TestAdd_retry_logic_distinguishes_errors(t *testing.T) { t.Parallel() // Server returns valid data after 2 seconds callCount := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { callCount++ if callCount == 1 { time.Sleep(2 * time.Second) // First call times out } w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // First attempt: times out resource1, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) ctx1, cancel1 := context.WithTimeout(ctx, 500*time.Millisecond) defer cancel1() err1 := ctrl.Add(ctx1, resource1) require.ErrorIs(t, err1, httprc.ErrNotReady(), "first attempt should return ErrNotReady") // Verify resource is in backend - should NOT retry Add() existing, lookupErr := ctrl.Lookup(ctx, server.URL) require.NoError(t, lookupErr, "resource should be in backend") require.NotNil(t, existing) // Second attempt: should fail with duplicate URL (correct behavior) resource2, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) err2 := ctrl.Add(ctx, resource2) require.Error(t, err2, "second Add should fail with duplicate URL") require.NotErrorIs(t, err2, httprc.ErrNotReady(), "duplicate URL should not return ErrNotReady") // Instead, wait for existing resource to be ready ctx3, cancel3 := context.WithTimeout(ctx, 5*time.Second) defer cancel3() err3 := existing.Ready(ctx3) require.NoError(t, err3, "existing resource should become ready") } // controlledHandler implements a deterministic HTTP handler for testing // retry logic. It uses an atomic counter to control exactly when to // transition from failure (invalid JSON) to success (valid JSON). // See DESIGN_SYNC_TEST.md for detailed design rationale. type controlledHandler struct { failuresRemaining atomic.Int32 } func (h *controlledHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) // Atomically decrement and check if we should still fail if remaining := h.failuresRemaining.Add(-1); remaining >= 0 { // Still have failures remaining, return invalid JSON w.Write([]byte(`{invalid json`)) } else { // No more failures needed, return valid JSON json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } } // TestIntegration_invalid_json_background_retry tests the end-to-end flow // when a server initially returns invalid JSON, then valid JSON on retry func TestIntegration_invalid_json_background_retry(t *testing.T) { t.Parallel() // Test configuration const ( minInterval = 100 * time.Millisecond timeout = 500 * time.Millisecond ) // Validate preconditions require.Greater(t, timeout, minInterval, "timeout must be greater than minInterval to allow at least one fetch") // Setup controlled server // Calculate failures needed with 2x safety margin to guarantee timeout // Formula: (timeout / minInterval) * 2 // Note: Integer division truncates, making this calculation conservative // Example: (500ms / 100ms) * 2 = 5 * 2 = 10 failures failuresNeeded := int32((timeout / minInterval) * 2) handler := &controlledHandler{} handler.failuresRemaining.Store(failuresNeeded) server := httptest.NewServer(handler) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), httprc.WithMinInterval(minInterval), ) require.NoError(t, err) // Add resource with timeout // Server is configured to fail enough times that timeout will fire addCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() err = ctrl.Add(addCtx, resource) // Assert ErrNotReady returned // Guaranteed because server will keep failing until we change the counter require.ErrorIs(t, err, httprc.ErrNotReady(), "expected ErrNotReady") // Verify resource is in backend (registration succeeded) existing, lookupErr := ctrl.Lookup(ctx, server.URL) require.NoError(t, lookupErr, "resource should be in backend") require.NotNil(t, existing, "resource should exist") // Allow server to succeed on next fetch // Safe to modify counter now because: // - Previous fetch failed (we got ErrNotReady) // - Next fetch won't start until minInterval elapses // - No concurrent access to the counter at this moment handler.failuresRemaining.Store(0) // Wait for background retry to make resource ready // The next fetch attempt will succeed and close the ready channel readyCtx, readyCancel := context.WithTimeout(ctx, 5*time.Second) defer readyCancel() err = resource.Ready(readyCtx) require.NoError(t, err, "resource should become ready after background retry") // Verify data is available var data map[string]string err = resource.Get(&data) require.NoError(t, err, "should be able to get data") require.Equal(t, "ok", data["status"], "data should have correct status") } // TestGH1551 reproduces the deadlock described in // https://github.com/lestrrat-go/jwx/issues/1551. // // When manual Refresh() calls fail (e.g. HTTP 500), each failure kills the // worker goroutine that processed it (worker.go returns on sync error). // After N failures (where N = number of workers), all workers are dead and // subsequent Refresh() calls block forever because nobody drains the // syncoutgoing channel. func TestGH1551(t *testing.T) { t.Parallel() t.Run("sync refresh deadlock", func(t *testing.T) { t.Parallel() const numWorkers = 3 // Server that always returns HTTP 500 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "Internal Server Error", http.StatusInternalServerError) })) t.Cleanup(srv.Close) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } cl := httprc.NewClient( httprc.WithWorkers(numWorkers), httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Register a resource without waiting for it to become ready // (since it will never succeed with a 500 server) resource, err := httprc.NewResource[[]byte]( srv.URL, httprc.BytesTransformer(), httprc.WithConstantInterval(time.Hour), // prevent periodic refresh from interfering ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false))) // Fire numWorkers Refresh() calls that all fail. Each one kills a worker. for i := range numWorkers { refreshCtx, refreshCancel := context.WithTimeout(ctx, 5*time.Second) err := ctrl.Refresh(refreshCtx, srv.URL) refreshCancel() // The refresh should return an error (HTTP 500), not hang. require.Error(t, err, "Refresh #%d should return an error", i) t.Logf("Refresh #%d failed as expected: %v", i, err) } // At this point all workers should be dead (before the fix). // The next Refresh() call will deadlock because no worker is alive // to drain the syncoutgoing channel. t.Log("All workers should have failed. Attempting one more Refresh()...") deadlockCtx, deadlockCancel := context.WithTimeout(ctx, 5*time.Second) defer deadlockCancel() err = ctrl.Refresh(deadlockCtx, srv.URL) // Before the fix: this blocks until deadlockCtx expires (context.DeadlineExceeded). // After the fix: this should return promptly with the HTTP 500 error. require.Error(t, err, "Refresh after all workers failed should still return an error") require.NotErrorIs(t, err, context.DeadlineExceeded, "Refresh should not deadlock (got context deadline exceeded, indicating workers are stuck)") t.Logf("Post-failure Refresh returned: %v", err) }) // Verify that periodic (async) refresh continues to function even after // synchronous Refresh() calls have failed. Before the fix, dead workers // meant async refreshes also stopped being processed. t.Run("async refresh still works after sync failures", func(t *testing.T) { t.Parallel() const numWorkers = 2 // Server that starts returning 500, then switches to 200 var shouldFail atomic.Bool shouldFail.Store(true) var successCount atomic.Int64 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { if shouldFail.Load() { http.Error(w, "Internal Server Error", http.StatusInternalServerError) return } successCount.Add(1) w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("ok")) })) t.Cleanup(srv.Close) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } cl := httprc.NewClient( httprc.WithWorkers(numWorkers), httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[[]byte]( srv.URL, httprc.BytesTransformer(), httprc.WithConstantInterval(500*time.Millisecond), // short interval for async refresh ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false))) // Let the initial async fetch (triggered by Add) complete before // starting sync refreshes, to avoid a race between the async // dispatch and the synchronous Refresh calls. time.Sleep(time.Second) // Kill all workers via failed sync refreshes for i := range numWorkers { refreshCtx, refreshCancel := context.WithTimeout(ctx, 5*time.Second) err := ctrl.Refresh(refreshCtx, srv.URL) refreshCancel() require.Error(t, err, "Refresh #%d should fail", i) } // Now make the server return 200 shouldFail.Store(false) countBefore := successCount.Load() // Wait for async refreshes to pick up the resource time.Sleep(3 * time.Second) countAfter := successCount.Load() require.Greater(t, countAfter, countBefore, "Async refresh should still work after sync refresh failures killed workers. "+ "No successful requests were made, indicating workers are dead.") }) } // TestIntegration_multiple_ready_calls_after_err_not_ready tests that multiple // Ready() calls work correctly after ErrNotReady func TestIntegration_multiple_ready_calls_after_err_not_ready(t *testing.T) { t.Parallel() // Setup: Server returns invalid JSON first, then valid JSON callCount := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { callCount++ w.WriteHeader(http.StatusOK) if callCount == 1 { w.Write([]byte(`{invalid json`)) // First call: invalid } else { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } })) t.Cleanup(server.Close) ctx := context.Background() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]string]( server.URL, httprc.JSONTransformer[map[string]string](), httprc.WithMinInterval(100*time.Millisecond), ) require.NoError(t, err) // First Add() - returns ErrNotReady ctx1, cancel1 := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel1() err1 := ctrl.Add(ctx1, resource) require.ErrorIs(t, err1, httprc.ErrNotReady(), "expected ErrNotReady") // Verify resource is in backend existing, lookupErr := ctrl.Lookup(ctx, server.URL) require.NoError(t, lookupErr, "resource should be in backend") require.NotNil(t, existing) // Call Ready() again - should eventually succeed after backend retries ctx2, cancel2 := context.WithTimeout(ctx, 5*time.Second) defer cancel2() err2 := resource.Ready(ctx2) require.NoError(t, err2, "second Ready() should succeed after backend retry") // Verify data is now available var data1 map[string]string err = resource.Get(&data1) require.NoError(t, err, "should be able to get data") require.Equal(t, "ok", data1["status"]) // Subsequent Ready() calls should succeed immediately ctx3, cancel3 := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel3() err3 := existing.Ready(ctx3) require.NoError(t, err3, "subsequent Ready() should succeed immediately") // Data should still be available var data2 map[string]string err = existing.Get(&data2) require.NoError(t, err, "should still be able to get data") require.Equal(t, "ok", data2["status"]) } // TestPeriodicCheckDeadlock reproduces the deadlock described in // https://github.com/lestrrat-go/httprc/issues/113 // // The deadlock occurs when: // 1. A periodic check in the controller iterates ready resources and // queues work for them by sending items on c.outgoing. // 2. c.outgoing is a buffered channel with capacity numWorkers+1. With // numWorkers=1, it can hold 2 items. With 1 worker and N>2 ready // resources, the controller eventually blocks trying to send the 3rd // item to c.outgoing once the buffer is full. // 3. The worker that picked up the 1st item finishes and attempts to send // a control message (such as an interval adjustment) to w.incoming // (= c.incoming). // 4. But c.incoming is read by the controller loop, which is currently // blocked trying to send to c.outgoing. // 5. Circular wait → deadlock. // // The test registers multiple resources with 1 worker and short refresh // intervals, waits for a periodic check to fire, then attempts to register // a new resource (which also sends to c.incoming). If the deadlock is // present, the new registration will hang until the context deadline. func TestPeriodicCheckDeadlock(t *testing.T) { t.Parallel() const ( numResources = 6 numWorkers = 1 minRefreshInterval = time.Second maxRefreshInterval = 2 * time.Second ) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"status":"ok"}`)) })) t.Cleanup(srv.Close) ctx, cancel := context.WithCancel(context.Background()) defer cancel() traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } cl := httprc.NewClient( httprc.WithWorkers(numWorkers), httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Register N resources, all with the same short refresh interval. // Each resource uses a distinct URL path so they are treated as separate items. for i := range numResources { url := srv.URL + "/resource/" + strconv.Itoa(i) r, err := httprc.NewResource[map[string]string]( url, httprc.JSONTransformer[map[string]string](), httprc.WithMinInterval(minRefreshInterval), httprc.WithMaxInterval(maxRefreshInterval), ) require.NoError(t, err, "NewResource should succeed for resource %d", i) addCtx, addCancel := context.WithTimeout(ctx, 10*time.Second) err = ctrl.Add(addCtx, r) addCancel() require.NoError(t, err, "Add should succeed for resource %d", i) } // Wait long enough for all resources to become due for periodic refresh. // maxRefreshInterval + 1s gives headroom for the tick to fire. time.Sleep(maxRefreshInterval + time.Second) // Now attempt to register a new resource. This sends an addRequest to // c.incoming. If the deadlock is present, periodicCheck is blocked // sending to c.outgoing while the worker is blocked sending to // c.incoming, so this Add will hang. newURL := srv.URL + "/resource/new" newR, err := httprc.NewResource[map[string]string]( newURL, httprc.JSONTransformer[map[string]string](), httprc.WithMinInterval(minRefreshInterval), httprc.WithMaxInterval(maxRefreshInterval), ) require.NoError(t, err, "NewResource should succeed for new resource") addCtx, addCancel := context.WithTimeout(ctx, 5*time.Second) defer addCancel() // Use WithWaitReady(false) so that a timeout here can only be caused by // the controller deadlock, not by a slow initial fetch under backlog. err = ctrl.Add(addCtx, newR, httprc.WithWaitReady(false)) // Before fix: context.DeadlineExceeded (Add hangs for 5s, then times out) // After fix: succeeds promptly require.NoError(t, err, "Add should not deadlock; got timeout indicating periodicCheck deadlock (issue #113)") // Verify the new resource is accessible existing, lookupErr := ctrl.Lookup(ctx, newURL) require.NoError(t, lookupErr, "Lookup should find the newly added resource") require.Equal(t, newURL, existing.URL()) } golang-github-lestrrat-go-httprc-3.0.5/integration_test.go000066400000000000000000000165751516451143700237470ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "sync" "sync/atomic" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/lestrrat-go/httprc/v3/errsink" "github.com/lestrrat-go/httprc/v3/tracesink" "github.com/stretchr/testify/require" ) func TestErrorSinkIntegration(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() var capturedErrors []error var mu sync.Mutex errorSink := errsink.NewFunc(func(_ context.Context, err error) { mu.Lock() defer mu.Unlock() capturedErrors = append(capturedErrors, err) }) // Create a server that returns errors errorSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "Server Error", http.StatusInternalServerError) })) defer errorSrv.Close() cl := httprc.NewClient(httprc.WithErrorSink(errorSink)) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[[]byte]( errorSrv.URL, httprc.BytesTransformer(), ) require.NoError(t, err) // Add resource without waiting for ready (to avoid test blocking) err = ctrl.Add(ctx, resource, httprc.WithWaitReady(false)) require.NoError(t, err) // Wait a bit for error to be captured time.Sleep(500 * time.Millisecond) mu.Lock() errorCount := len(capturedErrors) mu.Unlock() require.Positive(t, errorCount, "should have captured at least one error") } func TestTraceSinkIntegration(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() var capturedTraces []string var mu sync.Mutex traceSink := tracesink.Func(func(_ context.Context, msg string) { mu.Lock() defer mu.Unlock() capturedTraces = append(capturedTraces, msg) }) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(func() { srv.Close() }) cl := httprc.NewClient(httprc.WithTraceSink(traceSink)) ctrl, err := cl.Start(ctx) require.NoError(t, err) resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NoError(t, ctrl.Add(ctx, resource), "adding trace test resource should succeed") // Wait a bit for traces to be generated time.Sleep(time.Second) ctrl.Shutdown(time.Second) mu.Lock() traceCount := len(capturedTraces) mu.Unlock() require.Positive(t, traceCount, "should have captured trace messages") // Check for expected trace messages mu.Lock() traces := make([]string, len(capturedTraces)) copy(traces, capturedTraces) mu.Unlock() foundControllerStart := false foundResourceAdded := false for _, trace := range traces { t.Logf("Captured trace: %q", trace) switch trace { case "httprc controller: starting main controller loop": foundControllerStart = true case fmt.Sprintf("httprc controller: added resource %q", srv.URL): foundResourceAdded = true } } require.True(t, foundControllerStart, "should have trace for controller start") require.True(t, foundResourceAdded, "should have trace for resource addition") } func TestConcurrentResourceAccess(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() var requestCount int64 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { count := atomic.AddInt64(&requestCount, 1) json.NewEncoder(w).Encode(map[string]int64{"count": count}) })) defer srv.Close() cl := httprc.NewClient(httprc.WithWorkers(10)) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Add multiple resources const numResources = 5 resources := make([]httprc.Resource, numResources) for i := range numResources { resource, err := httprc.NewResource[map[string]int64]( fmt.Sprintf("%s/resource-%d", srv.URL, i), httprc.JSONTransformer[map[string]int64](), ) require.NoError(t, err) resources[i] = resource err = ctrl.Add(ctx, resource) require.NoError(t, err) } // Concurrent access to resources const numGoroutines = 20 const numOperations = 10 var wg sync.WaitGroup for i := range numGoroutines { wg.Add(1) go func(workerID int) { defer wg.Done() for j := range numOperations { resourceIdx := (workerID + j) % numResources resource := resources[resourceIdx] var data map[string]int64 err := resource.Get(&data) if err != nil { t.Errorf("worker %d: failed to get data from resource %d: %v", workerID, resourceIdx, err) return } if data["count"] <= 0 { t.Errorf("worker %d: invalid count %d from resource %d", workerID, data["count"], resourceIdx) return } // Trigger refresh occasionally if j%3 == 0 { err := ctrl.Refresh(ctx, resource.URL()) if err != nil { t.Errorf("worker %d: failed to refresh resource %d: %v", workerID, resourceIdx, err) return } } } }(i) } wg.Wait() // Verify final state for i, resource := range resources { var data map[string]int64 err := resource.Get(&data) require.NoError(t, err, "resource %d should be accessible", i) require.Positive(t, data["count"], "resource %d should have valid count", i) } } func TestResourceLeaks(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) defer cancel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) defer srv.Close() // Test that adding and removing many resources doesn't cause leaks const cycles = 10 const resourcesPerCycle = 20 for cycle := range cycles { cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) // Add many resources urls := make([]string, resourcesPerCycle) for i := range resourcesPerCycle { testURL := fmt.Sprintf("%s/leak-test-%d-%d", srv.URL, cycle, i) urls[i] = testURL resource, err := httprc.NewResource[map[string]string]( testURL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) err = ctrl.Add(ctx, resource) require.NoError(t, err) } // Remove half of them for i := range resourcesPerCycle / 2 { err := ctrl.Remove(ctx, urls[i]) require.NoError(t, err) } // Shutdown cleanly err = ctrl.Shutdown(time.Second) require.NoError(t, err) } } func TestWhitelistIntegration(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) t.Cleanup(srv.Close) t.Run("insecure whitelist allows all", func(t *testing.T) { t.Parallel() cl := httprc.NewClient(httprc.WithWhitelist(httprc.NewInsecureWhitelist())) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Should allow any URL resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) err = ctrl.Add(ctx, resource) require.NoError(t, err) }) // Note: Testing restrictive whitelists would require implementing // a custom whitelist type, which is outside the scope of this test } golang-github-lestrrat-go-httprc-3.0.5/options.go000066400000000000000000000104061516451143700220430ustar00rootroot00000000000000package httprc import ( "time" "github.com/lestrrat-go/option/v2" ) type NewClientOption interface { option.Interface newClientOption() } type newClientOption struct { option.Interface } func (newClientOption) newClientOption() {} type identWorkers struct{} // WithWorkers specifies the number of concurrent workers to use for the client. // If n is less than or equal to 0, the client will use a single worker. func WithWorkers(n int) NewClientOption { return newClientOption{option.New(identWorkers{}, n)} } type identErrorSink struct{} // WithErrorSink specifies the error sink to use for the client. // If not specified, the client will use a NopErrorSink. func WithErrorSink(sink ErrorSink) NewClientOption { return newClientOption{option.New(identErrorSink{}, sink)} } type identTraceSink struct{} // WithTraceSink specifies the trace sink to use for the client. // If not specified, the client will use a NopTraceSink. func WithTraceSink(sink TraceSink) NewClientOption { return newClientOption{option.New(identTraceSink{}, sink)} } type identWhitelist struct{} // WithWhitelist specifies the whitelist to use for the client. // If not specified, the client will use a BlockAllWhitelist. func WithWhitelist(wl Whitelist) NewClientOption { return newClientOption{option.New(identWhitelist{}, wl)} } type NewResourceOption interface { option.Interface newResourceOption() } type newResourceOption struct { option.Interface } func (newResourceOption) newResourceOption() {} type NewClientResourceOption interface { option.Interface newResourceOption() newClientOption() } type newClientResourceOption struct { option.Interface } func (newClientResourceOption) newResourceOption() {} func (newClientResourceOption) newClientOption() {} type identHTTPClient struct{} // WithHTTPClient specifies the HTTP client to use for the client. // If not specified, the client will use http.DefaultClient. // // This option can be passed to NewClient or NewResource. func WithHTTPClient(cl HTTPClient) NewClientResourceOption { return newClientResourceOption{option.New(identHTTPClient{}, cl)} } type identMinimumInterval struct{} // WithMinInterval specifies the minimum interval between fetches. // // This option affects the dynamic calculation of the interval between fetches. // If the value calculated from the http.Response is less than the this value, // the client will use this value instead. func WithMinInterval(d time.Duration) NewResourceOption { return newResourceOption{option.New(identMinimumInterval{}, d)} } type identMaximumInterval struct{} // WithMaxInterval specifies the maximum interval between fetches. // // This option affects the dynamic calculation of the interval between fetches. // If the value calculated from the http.Response is greater than the this value, // the client will use this value instead. func WithMaxInterval(d time.Duration) NewResourceOption { return newResourceOption{option.New(identMaximumInterval{}, d)} } type identConstantInterval struct{} // WithConstantInterval specifies the interval between fetches. When you // specify this option, the client will fetch the resource at the specified // intervals, regardless of the response's Cache-Control or Expires headers. // // By default this option is disabled. func WithConstantInterval(d time.Duration) NewResourceOption { return newResourceOption{option.New(identConstantInterval{}, d)} } type AddOption interface { option.Interface newAddOption() } type addOption struct { option.Interface } func (addOption) newAddOption() {} type identWaitReady struct{} // WithWaitReady specifies whether the client should wait for the resource to be // ready before returning from the Add method. // // By default, the client will wait for the resource to be ready before returning. // If you specify this option with a value of false, the client will not wait for // the resource to be fully registered, which is usually not what you want. // This option exists to accommodate for cases where you for some reason want to // add a resource to the controller, but want to do something else before // you wait for it. Make sure to call `r.Ready()` later on to ensure that // the resource is ready before you try to access it. func WithWaitReady(b bool) AddOption { return addOption{option.New(identWaitReady{}, b)} } golang-github-lestrrat-go-httprc-3.0.5/proxysink/000077500000000000000000000000001516451143700220665ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/proxysink/proxysink.go000066400000000000000000000043171516451143700244700ustar00rootroot00000000000000package proxysink import ( "context" "sync" ) type Backend[T any] interface { Put(context.Context, T) } // Proxy is used to send values through a channel. This is used to // serialize calls to underlying sinks. type Proxy[T any] struct { mu *sync.Mutex cancel context.CancelFunc ch chan T cond *sync.Cond pending []T backend Backend[T] closed bool } func New[T any](b Backend[T]) *Proxy[T] { mu := &sync.Mutex{} return &Proxy[T]{ ch: make(chan T, 1), mu: mu, cond: sync.NewCond(mu), backend: b, cancel: func() {}, } } func (p *Proxy[T]) Run(ctx context.Context) { defer p.cond.Broadcast() p.mu.Lock() ctx, cancel := context.WithCancel(ctx) p.cancel = cancel p.mu.Unlock() go p.controlloop(ctx) go p.flushloop(ctx) <-ctx.Done() } func (p *Proxy[T]) controlloop(ctx context.Context) { defer p.cond.Broadcast() for { select { case <-ctx.Done(): return case r := <-p.ch: p.mu.Lock() p.pending = append(p.pending, r) p.mu.Unlock() } p.cond.Broadcast() } } func (p *Proxy[T]) flushloop(ctx context.Context) { const defaultPendingSize = 10 pending := make([]T, defaultPendingSize) for { select { case <-ctx.Done(): p.mu.Lock() if len(p.pending) <= 0 { p.mu.Unlock() return } p.mu.Unlock() default: } p.mu.Lock() for len(p.pending) <= 0 { select { case <-ctx.Done(): p.mu.Unlock() return default: p.cond.Wait() } } // extract all pending values, and clear the shared slice if cap(pending) < len(p.pending) { pending = make([]T, len(p.pending)) } else { pending = pending[:len(p.pending)] } copy(pending, p.pending) if cap(p.pending) > defaultPendingSize { p.pending = make([]T, 0, defaultPendingSize) } else { p.pending = p.pending[:0] } p.mu.Unlock() for _, v := range pending { // send to sink serially p.backend.Put(ctx, v) } } } func (p *Proxy[T]) Put(ctx context.Context, v T) { p.mu.Lock() if p.closed { p.mu.Unlock() return } p.mu.Unlock() select { case <-ctx.Done(): return case p.ch <- v: return } } func (p *Proxy[T]) Close() { p.mu.Lock() defer p.mu.Unlock() if !p.closed { p.closed = true } p.cancel() p.cond.Broadcast() } golang-github-lestrrat-go-httprc-3.0.5/proxysink/proxysink_test.go000066400000000000000000000262071516451143700255310ustar00rootroot00000000000000package proxysink_test import ( "context" "fmt" "slices" "sync" "testing" "time" "github.com/lestrrat-go/httprc/v3/proxysink" ) type mockBackend[T any] struct { mu sync.Mutex puts []T } func (m *mockBackend[T]) Put(_ context.Context, v T) { m.mu.Lock() defer m.mu.Unlock() m.puts = append(m.puts, v) } func (m *mockBackend[T]) getPuts() []T { m.mu.Lock() defer m.mu.Unlock() result := make([]T, len(m.puts)) copy(result, m.puts) return result } func TestProxyBasic(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put some values proxy.Put(ctx, "test1") proxy.Put(ctx, "test2") proxy.Put(ctx, "test3") // Give some time for processing time.Sleep(50 * time.Millisecond) puts := backend.getPuts() if len(puts) != 3 { t.Errorf("expected 3 puts, got %d", len(puts)) return } expected := []string{"test1", "test2", "test3"} for i, exp := range expected { if puts[i] != exp { t.Errorf("put %d: expected %q, got %q", i, exp, puts[i]) } } } func TestProxyWithCancelledContext(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) cancel() // Cancel immediately // Should not block or panic proxy.Put(ctx, "test") // Give some time time.Sleep(10 * time.Millisecond) puts := backend.getPuts() if len(puts) != 0 { t.Errorf("expected 0 puts with cancelled context, got %d", len(puts)) } } func TestProxyMultipleValues(t *testing.T) { t.Parallel() backend := &mockBackend[int]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put many values rapidly const numValues = 100 for i := range numValues { proxy.Put(ctx, i) } // Give time for processing time.Sleep(100 * time.Millisecond) puts := backend.getPuts() if len(puts) != numValues { t.Errorf("expected %d puts, got %d", numValues, len(puts)) return } // Values should be in order for i := range numValues { if puts[i] != i { t.Errorf("put %d: expected %d, got %d", i, i, puts[i]) } } } func TestProxyContextCancellation(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put some values proxy.Put(ctx, "before_cancel") // Give time for processing time.Sleep(20 * time.Millisecond) // Cancel context cancel() // Give time for cleanup time.Sleep(20 * time.Millisecond) // Try to put after cancellation proxy.Put(ctx, "after_cancel") // Give more time time.Sleep(20 * time.Millisecond) puts := backend.getPuts() // Should have at least the value before cancellation if len(puts) == 0 { t.Error("expected at least one put before cancellation") return } if puts[0] != "before_cancel" { t.Errorf("expected first put to be %q, got %q", "before_cancel", puts[0]) } } func TestProxyWithTimeout(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) // Short timeout context ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) proxy.Put(ctx, "test_timeout") // Wait for timeout time.Sleep(60 * time.Millisecond) puts := backend.getPuts() if len(puts) != 1 { t.Errorf("expected 1 put, got %d", len(puts)) return } if puts[0] != "test_timeout" { t.Errorf("expected %q, got %q", "test_timeout", puts[0]) } } func TestProxyInterface(t *testing.T) { t.Parallel() // Ensure Proxy implements Backend interface backend := &mockBackend[string]{} var _ proxysink.Backend[string] = proxysink.New(backend) } func TestProxyBatching(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put values very rapidly to test batching behavior values := []string{"a", "b", "c", "d", "e"} for _, v := range values { proxy.Put(ctx, v) } // Give time for processing time.Sleep(100 * time.Millisecond) puts := backend.getPuts() if len(puts) != len(values) { t.Errorf("expected %d puts, got %d", len(values), len(puts)) return } // All values should be present for i, expected := range values { if puts[i] != expected { t.Errorf("put %d: expected %q, got %q", i, expected, puts[i]) } } } func TestProxyClose(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) // Test Close function - this was not covered at all (0%) proxy.Close() // Try to put after close - this will panic, so we need to catch it ctx := context.Background() func() { defer func() { if r := recover(); r != nil { // Expected panic due to sending on closed channel t.Logf("Expected panic caught: %v", r) } }() proxy.Put(ctx, "should_panic") }() // Give some time time.Sleep(10 * time.Millisecond) puts := backend.getPuts() // Should be empty since channel is closed and Put panicked if len(puts) != 0 { t.Errorf("expected 0 puts after close, got %d", len(puts)) } } func TestProxyPutWithCancelledContext(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) // Create already cancelled context ctx, cancel := context.WithCancel(context.Background()) cancel() // This should hit the ctx.Done() case in Put function proxy.Put(ctx, "test_cancelled") // Give some time time.Sleep(10 * time.Millisecond) puts := backend.getPuts() if len(puts) != 0 { t.Errorf("expected 0 puts with cancelled context, got %d", len(puts)) } } func TestProxyFlushLoopCancellation(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put a value proxy.Put(ctx, "test_value") // Give time for processing time.Sleep(20 * time.Millisecond) // Cancel context while there might be pending values cancel() // Give time for cleanup time.Sleep(50 * time.Millisecond) puts := backend.getPuts() if len(puts) != 1 { t.Errorf("expected 1 put before cancellation, got %d", len(puts)) return } if puts[0] != "test_value" { t.Errorf("expected %q, got %q", "test_value", puts[0]) } } func TestProxyLargeBatch(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put a large number of values to test pending slice reallocation const numValues = 50 for i := range numValues { proxy.Put(ctx, fmt.Sprintf("value_%d", i)) } // Give time for processing time.Sleep(200 * time.Millisecond) puts := backend.getPuts() if len(puts) != numValues { t.Errorf("expected %d puts, got %d", numValues, len(puts)) return } // Check all values are present and in order for i := range numValues { expected := fmt.Sprintf("value_%d", i) if puts[i] != expected { t.Errorf("put %d: expected %q, got %q", i, expected, puts[i]) } } } func TestProxyChannelBlocking(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Don't start the Run goroutine to make channel block // This should test the blocking behavior in Put // Try to put - this should not block indefinitely due to buffered channel proxy.Put(ctx, "test_blocking") // Give some time time.Sleep(10 * time.Millisecond) // Now start the proxy go proxy.Run(ctx) // Give time for processing time.Sleep(50 * time.Millisecond) puts := backend.getPuts() if len(puts) != 1 { t.Errorf("expected 1 put, got %d", len(puts)) return } if puts[0] != "test_blocking" { t.Errorf("expected %q, got %q", "test_blocking", puts[0]) } } func TestProxyFlushLoopEmptyPendingOnCancel(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Cancel immediately without putting any values // This should test the case where flushloop receives ctx.Done() // with empty pending slice cancel() // Give time for cleanup time.Sleep(50 * time.Millisecond) puts := backend.getPuts() if len(puts) != 0 { t.Errorf("expected 0 puts with immediate cancellation, got %d", len(puts)) } } func TestProxyFlushLoopWithPendingOnCancel(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put multiple values rapidly to build up pending queue for i := range 5 { proxy.Put(ctx, fmt.Sprintf("pending_%d", i)) } // Cancel context while values are still in pending queue // This should test the case where ctx.Done() triggers but len(p.pending) > 0 cancel() // Give time for cleanup - should process remaining pending values time.Sleep(100 * time.Millisecond) puts := backend.getPuts() if len(puts) == 0 { t.Error("expected some puts to be processed before cancellation") } // All pending values should have been processed t.Logf("Processed %d values before cancellation", len(puts)) } func TestProxyLargePendingSliceReallocation(t *testing.T) { t.Parallel() backend := &mockBackend[string]{} proxy := proxysink.New(backend) ctx, cancel := context.WithCancel(context.Background()) defer cancel() go proxy.Run(ctx) // Give some time for goroutines to start time.Sleep(10 * time.Millisecond) // Put many values very rapidly to trigger pending slice growth beyond defaultPendingSize (10) // This should test the "if cap(p.pending) > defaultPendingSize" branch const numValues = 25 // Add all values very rapidly to increase chance of large pending queue for i := range numValues { proxy.Put(ctx, fmt.Sprintf("realloc_%d", i)) // Very short sleep to allow some batching if i%5 == 0 { time.Sleep(1 * time.Millisecond) } } // Give time for all processing time.Sleep(200 * time.Millisecond) puts := backend.getPuts() if len(puts) != numValues { t.Errorf("expected %d puts, got %d", numValues, len(puts)) return } // Check all values are present for i := range numValues { expected := fmt.Sprintf("realloc_%d", i) if !slices.Contains(puts, expected) { t.Errorf("missing expected value: %s", expected) } } } golang-github-lestrrat-go-httprc-3.0.5/resource.go000066400000000000000000000252171516451143700222050ustar00rootroot00000000000000package httprc import ( "context" "fmt" "io" "net/http" "net/url" "sync" "sync/atomic" "time" "github.com/lestrrat-go/blackmagic" "github.com/lestrrat-go/httpcc" "github.com/lestrrat-go/httprc/v3/tracesink" ) // ResourceBase is a generic Resource type type ResourceBase[T any] struct { u string ready chan struct{} // closed when the resource is ready (i.e. after first successful fetch) once sync.Once httpcl HTTPClient t Transformer[T] r atomic.Value next atomic.Value interval time.Duration minInterval atomic.Int64 maxInterval atomic.Int64 busy atomic.Bool } // NewResource creates a new Resource object which after fetching the // resource from the URL, will transform the response body using the // provided Transformer to an object of type T. // // This function will return an error if the URL is not a valid URL // (i.e. it cannot be parsed by url.Parse), or if the transformer is nil. func NewResource[T any](s string, transformer Transformer[T], options ...NewResourceOption) (*ResourceBase[T], error) { var httpcl HTTPClient var interval time.Duration minInterval := DefaultMinInterval maxInterval := DefaultMaxInterval for _, option := range options { switch option.Ident() { case identHTTPClient{}: if err := option.Value(&httpcl); err != nil { return nil, fmt.Errorf(`httprc.NewResource: failed to parse HTTPClient option: %w`, err) } case identMinimumInterval{}: if err := option.Value(&minInterval); err != nil { return nil, fmt.Errorf(`httprc.NewResource: failed to parse MinimumInterval option: %w`, err) } case identMaximumInterval{}: if err := option.Value(&maxInterval); err != nil { return nil, fmt.Errorf(`httprc.NewResource: failed to parse MaximumInterval option: %w`, err) } case identConstantInterval{}: if err := option.Value(&interval); err != nil { return nil, fmt.Errorf(`httprc.NewResource: failed to parse ConstantInterval option: %w`, err) } } } if transformer == nil { return nil, fmt.Errorf(`httprc.NewResource: %w`, errTransformerRequired) } if s == "" { return nil, fmt.Errorf(`httprc.NewResource: %w`, errURLCannotBeEmpty) } if _, err := url.Parse(s); err != nil { return nil, fmt.Errorf(`httprc.NewResource: %w`, err) } r := &ResourceBase[T]{ u: s, httpcl: httpcl, t: transformer, interval: interval, ready: make(chan struct{}), } if httpcl != nil { r.httpcl = httpcl } r.minInterval.Store(int64(minInterval)) r.maxInterval.Store(int64(maxInterval)) r.SetNext(time.Unix(0, 0)) // initially, it should be fetched immediately return r, nil } // URL returns the URL of the resource. func (r *ResourceBase[T]) URL() string { return r.u } // Ready returns an empty error when the resource is ready. If the context // is canceled before the resource is ready, it will return the error from // the context. func (r *ResourceBase[T]) Ready(ctx context.Context) error { select { case <-ctx.Done(): return ctx.Err() case <-r.ready: return nil } } // Get assigns the value of the resource to the provided pointer. // If using the `httprc.ResourceBase[T]` type directly, you can use the `Resource()` // method to get the resource directly. // // This method exists because parametric types cannot be assigned to a single object type // that return different return values of the specialized type. i.e. for resources // `ResourceBase[A]` and `ResourceBase[B]`, we cannot have a single interface that can // be assigned to the same interface type `X` that expects a `Resource()` method that // returns `A` or `B` depending on the type of the resource. When accessing the // resource through the `httprc.Resource` interface, use this method to obtain the // stored value. func (r *ResourceBase[T]) Get(dst any) error { return blackmagic.AssignIfCompatible(dst, r.Resource()) } // Resource returns the last fetched resource. If the resource has not been // fetched yet, this will return the zero value of type T. // // If you would rather wait until the resource is fetched, you can use the // `Ready()` method to wait until the resource is ready (i.e. fetched at least once). func (r *ResourceBase[T]) Resource() T { v := r.r.Load() switch v := v.(type) { case T: return v default: var zero T return zero } } func (r *ResourceBase[T]) Next() time.Time { //nolint:forcetypeassert return r.next.Load().(time.Time) } func (r *ResourceBase[T]) SetNext(v time.Time) { r.next.Store(v) } func (r *ResourceBase[T]) ConstantInterval() time.Duration { return r.interval } func (r *ResourceBase[T]) MaxInterval() time.Duration { return time.Duration(r.maxInterval.Load()) } func (r *ResourceBase[T]) MinInterval() time.Duration { return time.Duration(r.minInterval.Load()) } func (r *ResourceBase[T]) SetMaxInterval(v time.Duration) { r.maxInterval.Store(int64(v)) } func (r *ResourceBase[T]) SetMinInterval(v time.Duration) { r.minInterval.Store(int64(v)) } func (r *ResourceBase[T]) SetBusy(v bool) { r.busy.Store(v) } func (r *ResourceBase[T]) IsBusy() bool { return r.busy.Load() } // limitedBody is a wrapper around an io.Reader that will only read up to // MaxBufferSize bytes. This is provided to prevent the user from accidentally // reading a huge response body into memory type limitedBody struct { rdr io.Reader close func() error } func (l *limitedBody) Read(p []byte) (n int, err error) { return l.rdr.Read(p) } func (l *limitedBody) Close() error { return l.close() } type traceSinkKey struct{} func withTraceSink(ctx context.Context, sink TraceSink) context.Context { return context.WithValue(ctx, traceSinkKey{}, sink) } func traceSinkFromContext(ctx context.Context) TraceSink { if v := ctx.Value(traceSinkKey{}); v != nil { //nolint:forcetypeassert return v.(TraceSink) } return tracesink.Nop{} } type httpClientKey struct{} func withHTTPClient(ctx context.Context, cl HTTPClient) context.Context { return context.WithValue(ctx, httpClientKey{}, cl) } func httpClientFromContext(ctx context.Context) HTTPClient { if v := ctx.Value(httpClientKey{}); v != nil { //nolint:forcetypeassert return v.(HTTPClient) } return http.DefaultClient } func (r *ResourceBase[T]) Sync(ctx context.Context) error { traceSink := traceSinkFromContext(ctx) httpcl := r.httpcl if httpcl == nil { httpcl = httpClientFromContext(ctx) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.u, nil) if err != nil { return fmt.Errorf(`httprc.Resource.Sync: failed to create request: %w`, err) } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: fetching %q", r.u)) res, err := httpcl.Do(req) if err != nil { return fmt.Errorf(`httprc.Resource.Sync: failed to execute HTTP request: %w`, err) } defer res.Body.Close() next := r.calculateNextRefreshTime(ctx, res) traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: next refresh time for %q is %v", r.u, next)) r.SetNext(next) if res.StatusCode != http.StatusOK { return fmt.Errorf(`httprc.Resource.Sync: %w (status code=%d, url=%q)`, errUnexpectedStatusCode, res.StatusCode, r.u) } // replace the body of the response with a limited reader that // will only read up to MaxBufferSize bytes res.Body = &limitedBody{ rdr: &io.LimitedReader{R: res.Body, N: MaxBufferSize}, close: res.Body.Close, } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: transforming %q", r.u)) v, err := r.transform(ctx, res) if err != nil { return fmt.Errorf(`httprc.Resource.Sync: %w: %w`, errTransformerFailed, err) } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: storing new value for %q", r.u)) r.r.Store(v) r.once.Do(func() { close(r.ready) }) traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: stored value for %q", r.u)) return nil } func (r *ResourceBase[T]) transform(ctx context.Context, res *http.Response) (ret T, gerr error) { // Protect the call to Transform with a defer/recover block, so that even // if the Transform method panics, we can recover from it and return an error defer func() { if recovered := recover(); recovered != nil { gerr = fmt.Errorf(`httprc.Resource.transform: %w: %v`, errRecoveredFromPanic, recovered) } }() return r.t.Transform(ctx, res) } func (r *ResourceBase[T]) determineNextFetchInterval(ctx context.Context, name string, fromHeader, minValue, maxValue time.Duration) time.Duration { traceSink := traceSinkFromContext(ctx) if fromHeader > maxValue { traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s > maximum interval, using maximum interval %s", r.URL(), name, maxValue)) return maxValue } if fromHeader < minValue { traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s < minimum interval, using minimum interval %s", r.URL(), name, minValue)) return minValue } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Using %s (%s)", r.URL(), name, fromHeader)) return fromHeader } func (r *ResourceBase[T]) calculateNextRefreshTime(ctx context.Context, res *http.Response) time.Time { traceSink := traceSinkFromContext(ctx) now := time.Now() // If constant interval is set, use that regardless of what the // response headers say. if interval := r.ConstantInterval(); interval > 0 { traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Explicit interval set, using value %s", r.URL(), interval)) return now.Add(interval) } if interval := r.extractCacheControlMaxAge(ctx, res); interval > 0 { return now.Add(interval) } if interval := r.extractExpiresInterval(ctx, res); interval > 0 { return now.Add(interval) } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s No cache-control/expires headers found, using minimum interval", r.URL())) return now.Add(r.MinInterval()) } func (r *ResourceBase[T]) extractCacheControlMaxAge(ctx context.Context, res *http.Response) time.Duration { traceSink := traceSinkFromContext(ctx) v := res.Header.Get(`Cache-Control`) if v == "" { return 0 } dir, err := httpcc.ParseResponse(v) if err != nil { return 0 } maxAge, ok := dir.MaxAge() if !ok { return 0 } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Cache-Control=max-age directive set (%d)", r.URL(), maxAge)) return r.determineNextFetchInterval( ctx, "max-age", time.Duration(maxAge)*time.Second, r.MinInterval(), r.MaxInterval(), ) } func (r *ResourceBase[T]) extractExpiresInterval(ctx context.Context, res *http.Response) time.Duration { traceSink := traceSinkFromContext(ctx) v := res.Header.Get(`Expires`) if v == "" { return 0 } expires, err := http.ParseTime(v) if err != nil { return 0 } traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Expires header set (%s)", r.URL(), expires)) return r.determineNextFetchInterval( ctx, "expires", time.Until(expires), r.MinInterval(), r.MaxInterval(), ) } golang-github-lestrrat-go-httprc-3.0.5/resource_test.go000066400000000000000000000333241516451143700232420ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "errors" "io" "log/slog" "net/http" "net/http/httptest" "strings" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/lestrrat-go/httprc/v3/tracesink" "github.com/stretchr/testify/require" ) func TestResourceCreation(t *testing.T) { t.Parallel() t.Run("valid resource creation", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]string]( "https://example.com/test", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err) require.NotNil(t, resource) require.Equal(t, "https://example.com/test", resource.URL()) require.Equal(t, httprc.DefaultMinInterval, resource.MinInterval()) require.Equal(t, httprc.DefaultMaxInterval, resource.MaxInterval()) }) t.Run("resource with custom intervals", func(t *testing.T) { t.Parallel() minInterval := 30 * time.Second maxInterval := 2 * time.Hour resource, err := httprc.NewResource[map[string]string]( "https://example.com/test", httprc.JSONTransformer[map[string]string](), httprc.WithMinInterval(minInterval), httprc.WithMaxInterval(maxInterval), ) require.NoError(t, err) require.Equal(t, minInterval, resource.MinInterval()) require.Equal(t, maxInterval, resource.MaxInterval()) }) t.Run("resource with invalid URL", func(t *testing.T) { t.Parallel() // Test with malformed URLs invalidURLs := []string{ "", "not-a-url", "ftp://unsupported-scheme.com", "://missing-scheme", } for _, invalidURL := range invalidURLs { _, err := httprc.NewResource[map[string]string]( invalidURL, httprc.JSONTransformer[map[string]string](), ) // Note: The actual behavior depends on implementation // This test documents the current behavior if invalidURL == "" { require.Error(t, err, "empty URL should cause error") } } }) } func TestResourceTransformers(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/json": w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ "string": "test", "number": 42, "bool": true, }) case "/bytes": w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("binary data")) case "/text": w.Header().Set("Content-Type", "text/plain") w.Write([]byte("plain text")) case "/invalid-json": w.Header().Set("Content-Type", "application/json") w.Write([]byte("invalid json {")) } })) t.Cleanup(srv.Close) ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) traceDst := io.Discard if testing.Verbose() { traceDst = io.Discard } cl := httprc.NewClient( httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), ) ctrl, err := cl.Start(ctx) require.NoError(t, err, "client start should succeed") t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("JSON transformer", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]any]( srv.URL+"/json", httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err, "JSON resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding JSON resource should succeed") var data map[string]any require.NoError(t, resource.Get(&data), "getting JSON data should succeed") require.Equal(t, "test", data["string"]) require.InEpsilon(t, 42.0, data["number"], 1e-9) // JSON numbers are float64 require.Equal(t, true, data["bool"]) }) t.Run("bytes transformer", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[[]byte]( srv.URL+"/bytes", httprc.BytesTransformer(), ) require.NoError(t, err, "bytes resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding bytes resource should succeed") var data []byte require.NoError(t, resource.Get(&data), "getting bytes data should succeed") require.Equal(t, []byte("binary data"), data) }) t.Run("custom transformer", func(t *testing.T) { t.Parallel() customTransformer := httprc.TransformFunc[string](func(_ context.Context, res *http.Response) (string, error) { defer res.Body.Close() buf := make([]byte, 1024) n, _ := res.Body.Read(buf) return strings.ToUpper(string(buf[:n])), nil }) resource, err := httprc.NewResource[string]( srv.URL+"/text", customTransformer, ) require.NoError(t, err, "custom transformer resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding custom transformer resource should succeed") var data string require.NoError(t, resource.Get(&data), "getting custom transformed data should succeed") require.Equal(t, "PLAIN TEXT", data) }) t.Run("transformer error handling", func(t *testing.T) { t.Parallel() // JSON transformer should fail on invalid JSON resource, err := httprc.NewResource[map[string]any]( srv.URL+"/invalid-json", httprc.JSONTransformer[map[string]any](), ) require.NoError(t, err, "invalid JSON resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "should add resource without waiting for ready") // Ready should time out (and return the error) due to invalid JSON // (if the initial request fails, it should not block indefinitely) tctx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() require.Error(t, resource.Ready(tctx), "should not block indefinitely on invalid JSON") }) } func TestResourceErrorHandling(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err, "error handling test client start should succeed") t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("HTTP error responses", func(t *testing.T) { t.Parallel() errorSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/404": http.NotFound(w, r) case "/500": http.Error(w, "Internal Server Error", http.StatusInternalServerError) case "/timeout": time.Sleep(100 * time.Millisecond) // Simulate slow response w.Write([]byte("slow response")) } })) defer errorSrv.Close() // Test 404 error resource404, err := httprc.NewResource[[]byte]( errorSrv.URL+"/404", httprc.BytesTransformer(), ) require.NoError(t, err, "404 resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource404, httprc.WithWaitReady(false)), "adding 404 resource should succeed") readyCtx404, cancel404 := context.WithTimeout(ctx, time.Second) defer cancel404() require.Error(t, resource404.Ready(readyCtx404), "404 resource should not become ready") // Test 500 error resource500, err := httprc.NewResource[[]byte]( errorSrv.URL+"/500", httprc.BytesTransformer(), ) require.NoError(t, err, "500 resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource500, httprc.WithWaitReady(false)), "adding 500 resource should succeed") readyCtx500, cancel500 := context.WithTimeout(ctx, time.Second) defer cancel500() require.Error(t, resource500.Ready(readyCtx500), "500 resource should not become ready") }) t.Run("network error", func(t *testing.T) { // Use a non-existent server resource, err := httprc.NewResource[[]byte]( "http://127.0.0.1:99999/nonexistent", httprc.BytesTransformer(), ) require.NoError(t, err, "network error test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "adding network error test resource should succeed") // Ready should fail due to connection error readyCtx, cancel := context.WithTimeout(ctx, time.Second) defer cancel() require.Error(t, resource.Ready(readyCtx), "network error resource should not become ready") }) t.Run("context cancellation", func(t *testing.T) { slowSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) // Slow response w.Write([]byte("slow response")) })) defer slowSrv.Close() resource, err := httprc.NewResource[[]byte]( slowSrv.URL, httprc.BytesTransformer(), ) require.NoError(t, err, "context cancellation test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "adding context cancellation test resource should succeed") // Create a context with short timeout readyCtx, readyCancel := context.WithTimeout(ctx, 100*time.Millisecond) defer readyCancel() err = resource.Ready(readyCtx) require.Error(t, err) require.True(t, errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled)) }) } func TestResourceCacheHeaders(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) var requestCount int srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { requestCount++ switch r.URL.Path { case "/cache-control": w.Header().Set("Cache-Control", "max-age=1") json.NewEncoder(w).Encode(map[string]int{"count": requestCount}) case "/expires": w.Header().Set("Expires", time.Now().Add(1*time.Second).Format(http.TimeFormat)) json.NewEncoder(w).Encode(map[string]int{"count": requestCount}) case "/no-cache": w.Header().Set("Cache-Control", "no-cache") json.NewEncoder(w).Encode(map[string]int{"count": requestCount}) default: json.NewEncoder(w).Encode(map[string]int{"count": requestCount}) } })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err, "cache headers test client start should succeed") t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("respect cache-control max-age", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]int]( srv.URL+"/cache-control", httprc.JSONTransformer[map[string]int](), ) require.NoError(t, err, "cache control resource creation should succeed") // Set very short intervals to test cache behavior resource.SetMinInterval(100 * time.Millisecond) resource.SetMaxInterval(10 * time.Second) require.NoError(t, ctrl.Add(ctx, resource), "adding cache control resource should succeed") // Wait for cache to expire and resource to refresh time.Sleep(5 * time.Second) var data map[string]int require.NoError(t, resource.Get(&data), "getting cache control data should succeed") require.Greater(t, data["count"], 1, "resource should have been refreshed due to cache expiry") }) } func TestResourceReady(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ready"}) })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err, "ready test client start should succeed") t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("resource becomes ready", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]string]( srv.URL+"/ready-test", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "ready test resource creation should succeed") // Add without waiting for ready require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "adding ready test resource should succeed") // Now wait for it to become ready require.NoError(t, resource.Ready(ctx), "resource should become ready") var data map[string]string require.NoError(t, resource.Get(&data), "getting ready test data should succeed") require.Equal(t, "ready", data["status"]) }) t.Run("ready with timeout", func(t *testing.T) { t.Parallel() slowSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { time.Sleep(2 * time.Second) json.NewEncoder(w).Encode(map[string]string{"status": "slow"}) })) defer slowSrv.Close() resource, err := httprc.NewResource[map[string]string]( slowSrv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "ready timeout test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource, httprc.WithWaitReady(false)), "adding ready timeout test resource should succeed") // Ready should timeout readyCtx, readyCancel := context.WithTimeout(ctx, 100*time.Millisecond) defer readyCancel() err = resource.Ready(readyCtx) require.Error(t, err) require.ErrorIs(t, err, context.DeadlineExceeded) }) } func TestResourceIntervals(t *testing.T) { t.Parallel() t.Run("set and get intervals", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[[]byte]( "https://example.com/test", httprc.BytesTransformer(), ) require.NoError(t, err, "intervals test resource creation should succeed") // Test default values require.Equal(t, httprc.DefaultMinInterval, resource.MinInterval()) require.Equal(t, httprc.DefaultMaxInterval, resource.MaxInterval()) // Set new values newMin := 5 * time.Minute newMax := 2 * time.Hour resource.SetMinInterval(newMin) resource.SetMaxInterval(newMax) require.Equal(t, newMin, resource.MinInterval()) require.Equal(t, newMax, resource.MaxInterval()) }) t.Run("busy state", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[[]byte]( "https://example.com/test", httprc.BytesTransformer(), ) require.NoError(t, err, "busy state test resource creation should succeed") // Initially not busy require.False(t, resource.IsBusy()) // Set busy resource.SetBusy(true) require.True(t, resource.IsBusy()) // Unset busy resource.SetBusy(false) require.False(t, resource.IsBusy()) }) } golang-github-lestrrat-go-httprc-3.0.5/tools/000077500000000000000000000000001516451143700211605ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/tools/autodoc.pl000066400000000000000000000053001516451143700231510ustar00rootroot00000000000000#!perl use strict; use File::Temp; # Accept a list of filenames, and process them # if any of them has a diff, commit it # Use GITHUB_REF, but if the ref is develop/v\d, then use v\d my $link_ref = $ENV{GITHUB_REF}; $link_ref =~ s{^refs/heads/}{}; my @files = @ARGV; my @has_diff; for my $filename (@files) { open(my $src, '<', $filename) or die $!; my $output = File::Temp->new(SUFFIX => '.md'); my $skip_until_end; for my $line (<$src>) { if ($line =~ /^$/) { $skip_until_end = 0; } elsif ($skip_until_end) { next; } if ($line !~ /(^)$/) { $output->print($line); next; } $output->print("$1\n"); my $include_filename = $2; my $options = $3; $output->print("```go\n"); my $content = do { open(my $file, '<', $include_filename) or die "failed to include file $include_filename from source file $filename: $!"; local $/; <$file>; }; $content =~ s{^(\t+)}{" " x length($1)}gsme; $output->print($content); $output->print("```\n"); $output->print("source: [$include_filename](https://github.com/lestrrat-go/httprc/blob/$ENV{GITHUB_REF}/$include_filename)\n"); # now we need to skip copying until the end of INCLUDE $skip_until_end = 1; } $output->close(); close($src); if (!$ENV{AUTODOC_DRYRUN}) { rename $output->filename, $filename or die $!; my $diff = `git diff $filename`; if ($diff) { push @has_diff, $filename; } } } if (!$ENV{AUTODOC_DRYRUN}) { if (@has_diff) { # Write multi-line commit message in a file my $commit_message_file = File::Temp->new(SUFFIX => '.txt'); print $commit_message_file "autodoc updates\n\n"; print " - $_\n" for @has_diff; $commit_message_file->close(); system("git", "remote", "set-url", "origin", "https://github-actions:$ENV{GITHUB_TOKEN}\@github.com/$ENV{GITHUB_REPOSITORY}") == 0 or die $!; system("git", "config", "--global", "user.name", "$ENV{GITHUB_ACTOR}") == 0 or die $!; system("git", "config", "--global", "user.email", "$ENV{GITHUB_ACTOR}\@users.noreply.github.com") == 0 or die $!; system("git", "switch", "-c", "autodoc-pr-$ENV{GITHUB_HEAD_REF}") == 0 or die $!; system("git", "commit", "-F", $commit_message_file->filename, @files) == 0 or die $!; system("git", "push", "origin", "HEAD:autodoc-pr-$ENV{GITHUB_HEAD_REF}") == 0 or die $!; system("gh", "pr", "create", "--base", $link_ref, "--fill") == 0 or die $!; } } golang-github-lestrrat-go-httprc-3.0.5/tracesink/000077500000000000000000000000001516451143700220035ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-3.0.5/tracesink/tracesink.go000066400000000000000000000021341516451143700243150ustar00rootroot00000000000000package tracesink import ( "context" "log/slog" ) type Interface interface { Put(context.Context, string) } // Nop is an ErrorSink that does nothing. It does not require // any initialization, so the zero value can be used. type Nop struct{} // NewNop returns a new NopTraceSink object. The constructor // is provided for consistency. func NewNop() Interface { return Nop{} } // Put for NopTraceSink does nothing. func (Nop) Put(context.Context, string) {} type slogSink struct { level slog.Level logger SlogLogger } type SlogLogger interface { Log(context.Context, slog.Level, string, ...any) } // NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger func NewSlog(l SlogLogger) Interface { return &slogSink{ level: slog.LevelInfo, logger: l, } } func (s *slogSink) Put(ctx context.Context, v string) { s.logger.Log(ctx, s.level, v) } // Func is a TraceSink that calls a function with the trace message. type Func func(context.Context, string) // Put calls the function with the trace message. func (f Func) Put(ctx context.Context, msg string) { f(ctx, msg) } golang-github-lestrrat-go-httprc-3.0.5/tracesink/tracesink_test.go000066400000000000000000000070471516451143700253640ustar00rootroot00000000000000package tracesink_test import ( "context" "log/slog" "testing" "github.com/lestrrat-go/httprc/v3/tracesink" ) func TestNop(t *testing.T) { t.Parallel() tests := []struct { name string msg string }{ { name: "empty string", msg: "", }, { name: "simple message", msg: "test trace message", }, { name: "multiline message", msg: "line 1\nline 2\nline 3", }, { name: "message with special characters", msg: "test with %s formatting and special chars: 日本語", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() sink := tracesink.NewNop() // Should not panic or do anything ctx := context.Background() sink.Put(ctx, tt.msg) }) } } func TestNopZeroValue(t *testing.T) { t.Parallel() // Test that zero value can be used directly var sink tracesink.Nop ctx := context.Background() msg := "test trace message" // Should not panic sink.Put(ctx, msg) } type mockSlogger struct { logs []logEntry } type logEntry struct { level slog.Level msg string args []any } func (m *mockSlogger) Log(_ context.Context, level slog.Level, msg string, args ...any) { m.logs = append(m.logs, logEntry{ level: level, msg: msg, args: args, }) } func TestSlogSink(t *testing.T) { t.Parallel() tests := []struct { name string msg string wantMsg string wantArgs int }{ { name: "simple message", msg: "test trace message", wantMsg: "test trace message", wantArgs: 0, }, { name: "message with formatting chars", msg: "trace with %d number", wantMsg: "trace with %d number", wantArgs: 0, }, { name: "empty message", msg: "", wantMsg: "", wantArgs: 0, }, { name: "multiline message", msg: "line 1\nline 2", wantMsg: "line 1\nline 2", wantArgs: 0, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() logger := &mockSlogger{} sink := tracesink.NewSlog(logger) ctx := context.Background() sink.Put(ctx, tt.msg) if len(logger.logs) != 1 { t.Errorf("expected 1 log entry, got %d", len(logger.logs)) return } entry := logger.logs[0] if entry.level != slog.LevelInfo { t.Errorf("expected level %v, got %v", slog.LevelInfo, entry.level) } if entry.msg != tt.wantMsg { t.Errorf("expected message %q, got %q", tt.wantMsg, entry.msg) } if len(entry.args) != tt.wantArgs { t.Errorf("expected %d args, got %d", tt.wantArgs, len(entry.args)) } }) } } func TestSlogSinkMultipleMessages(t *testing.T) { t.Parallel() logger := &mockSlogger{} sink := tracesink.NewSlog(logger) ctx := context.Background() messages := []string{ "first trace message", "second trace message", "third trace message", } for _, msg := range messages { sink.Put(ctx, msg) } if len(logger.logs) != len(messages) { t.Errorf("expected %d log entries, got %d", len(messages), len(logger.logs)) return } for i, msg := range messages { if logger.logs[i].msg != msg { t.Errorf("log entry %d: expected message %q, got %q", i, msg, logger.logs[i].msg) } if logger.logs[i].level != slog.LevelInfo { t.Errorf("log entry %d: expected level %v, got %v", i, slog.LevelInfo, logger.logs[i].level) } } } func TestInterface(t *testing.T) { t.Parallel() // Ensure types implement the interface //nolint:staticcheck { var _ tracesink.Interface = (*tracesink.Nop)(nil) var _ tracesink.Interface = tracesink.NewNop() var _ tracesink.Interface = tracesink.NewSlog(&mockSlogger{}) } } golang-github-lestrrat-go-httprc-3.0.5/transformer.go000066400000000000000000000015761516451143700227220ustar00rootroot00000000000000package httprc import ( "context" "encoding/json" "io" "net/http" ) type bytesTransformer struct{} // BytesTransformer returns a Transformer that reads the entire response body // as a byte slice. This is the default Transformer used by httprc.Client func BytesTransformer() Transformer[[]byte] { return bytesTransformer{} } func (bytesTransformer) Transform(_ context.Context, res *http.Response) ([]byte, error) { return io.ReadAll(res.Body) } type jsonTransformer[T any] struct{} // JSONTransformer returns a Transformer that decodes the response body as JSON // into the provided type T. func JSONTransformer[T any]() Transformer[T] { return jsonTransformer[T]{} } func (jsonTransformer[T]) Transform(_ context.Context, res *http.Response) (T, error) { var v T if err := json.NewDecoder(res.Body).Decode(&v); err != nil { var zero T return zero, err } return v, nil } golang-github-lestrrat-go-httprc-3.0.5/whitelist.go000066400000000000000000000066251516451143700223740ustar00rootroot00000000000000package httprc import ( "regexp" "sync" ) // Whitelist is an interface that allows you to determine if a given URL is allowed // or not. Implementations of this interface can be used to restrict the URLs that // the client can access. // // By default all URLs are allowed, but this may not be ideal in production environments // for security reasons. // // This exists because you might use this module to store resources provided by // user of your application, in which case you cannot necessarily trust that the // URLs are safe. // // You will HAVE to provide some sort of whitelist. type Whitelist interface { IsAllowed(string) bool } // WhitelistFunc is a function type that implements the Whitelist interface. type WhitelistFunc func(string) bool func (f WhitelistFunc) IsAllowed(u string) bool { return f(u) } // BlockAllWhitelist is a Whitelist implementation that blocks all URLs. type BlockAllWhitelist struct{} // NewBlockAllWhitelist creates a new BlockAllWhitelist instance. It is safe to // use the zero value of this type; this constructor is provided for consistency. func NewBlockAllWhitelist() BlockAllWhitelist { return BlockAllWhitelist{} } func (BlockAllWhitelist) IsAllowed(_ string) bool { return false } // InsecureWhitelist is a Whitelist implementation that allows all URLs. Be careful // when using this in your production code: make sure you do not blindly register // URLs from untrusted sources. type InsecureWhitelist struct{} // NewInsecureWhitelist creates a new InsecureWhitelist instance. It is safe to // use the zero value of this type; this constructor is provided for consistency. func NewInsecureWhitelist() InsecureWhitelist { return InsecureWhitelist{} } func (InsecureWhitelist) IsAllowed(_ string) bool { return true } // RegexpWhitelist is a jwk.Whitelist object comprised of a list of *regexp.Regexp // objects. All entries in the list are tried until one matches. If none of the // *regexp.Regexp objects match, then the URL is deemed unallowed. type RegexpWhitelist struct { mu sync.RWMutex patterns []*regexp.Regexp } // NewRegexpWhitelist creates a new RegexpWhitelist instance. It is safe to use the // zero value of this type; this constructor is provided for consistency. func NewRegexpWhitelist() *RegexpWhitelist { return &RegexpWhitelist{} } // Add adds a new regular expression to the list of expressions to match against. func (w *RegexpWhitelist) Add(pat *regexp.Regexp) *RegexpWhitelist { w.mu.Lock() defer w.mu.Unlock() w.patterns = append(w.patterns, pat) return w } // IsAllowed returns true if any of the patterns in the whitelist // returns true. func (w *RegexpWhitelist) IsAllowed(u string) bool { w.mu.RLock() patterns := w.patterns w.mu.RUnlock() for _, pat := range patterns { if pat.MatchString(u) { return true } } return false } // MapWhitelist is a jwk.Whitelist object comprised of a map of strings. // If the URL exists in the map, then the URL is allowed to be fetched. type MapWhitelist interface { Whitelist Add(string) MapWhitelist } type mapWhitelist struct { mu sync.RWMutex store map[string]struct{} } func NewMapWhitelist() MapWhitelist { return &mapWhitelist{store: make(map[string]struct{})} } func (w *mapWhitelist) Add(pat string) MapWhitelist { w.mu.Lock() defer w.mu.Unlock() w.store[pat] = struct{}{} return w } func (w *mapWhitelist) IsAllowed(u string) bool { w.mu.RLock() _, b := w.store[u] w.mu.RUnlock() return b } golang-github-lestrrat-go-httprc-3.0.5/worker.go000066400000000000000000000035211516451143700216610ustar00rootroot00000000000000package httprc import ( "context" "fmt" "sync" ) type worker struct { httpcl HTTPClient incoming chan any next <-chan Resource nextsync <-chan synchronousRequest errSink ErrorSink traceSink TraceSink } func (w worker) Run(ctx context.Context, readywg *sync.WaitGroup, donewg *sync.WaitGroup) { w.traceSink.Put(ctx, "httprc worker: START worker loop") defer w.traceSink.Put(ctx, "httprc worker: END worker loop") defer donewg.Done() ctx = withTraceSink(ctx, w.traceSink) ctx = withHTTPClient(ctx, w.httpcl) readywg.Done() for { select { case <-ctx.Done(): w.traceSink.Put(ctx, "httprc worker: stopping worker loop") return case r := <-w.next: w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (async)", r.URL())) if err := r.Sync(ctx); err != nil { w.errSink.Put(ctx, err) } r.SetBusy(false) w.sendAdjustIntervalRequest(ctx, r) case sr := <-w.nextsync: w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (synchronous)", sr.resource.URL())) if err := sr.resource.Sync(ctx); err != nil { w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: FAILED to sync %q (synchronous): %s", sr.resource.URL(), err)) sendReply(ctx, sr.reply, struct{}{}, err) sr.resource.SetBusy(false) continue } w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: SUCCESS syncing %q (synchronous)", sr.resource.URL())) sr.resource.SetBusy(false) sendReply(ctx, sr.reply, struct{}{}, nil) w.sendAdjustIntervalRequest(ctx, sr.resource) } } } func (w worker) sendAdjustIntervalRequest(ctx context.Context, r Resource) { w.traceSink.Put(ctx, "httprc worker: Sending interval adjustment request for "+r.URL()) select { case <-ctx.Done(): case w.incoming <- adjustIntervalRequest{resource: r}: } w.traceSink.Put(ctx, "httprc worker: Sent interval adjustment request for "+r.URL()) } golang-github-lestrrat-go-httprc-3.0.5/worker_test.go000066400000000000000000000313201516451143700227160ustar00rootroot00000000000000package httprc_test import ( "context" "encoding/json" "fmt" "io" "log/slog" "net/http" "net/http/httptest" "os" "sync" "testing" "time" "github.com/lestrrat-go/httprc/v3" "github.com/lestrrat-go/httprc/v3/tracesink" "github.com/stretchr/testify/require" ) func TestWorkerPoolBehavior(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) var requestCount int64 var mu sync.Mutex var requestTimes []time.Time srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() requestTimes = append(requestTimes, time.Now()) requestCount++ count := requestCount mu.Unlock() // Simulate some work time.Sleep(10 * time.Millisecond) json.NewEncoder(w).Encode(map[string]int64{"count": count}) })) t.Cleanup(srv.Close) t.Run("worker pool processes requests concurrently", func(t *testing.T) { t.Parallel() const numWorkers = 5 traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } cl := httprc.NewClient( httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), httprc.WithWorkers(numWorkers), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Add multiple resources that will be fetched simultaneously const numResources = numWorkers * 2 for i := range numResources { resource, err := httprc.NewResource[map[string]int64]( fmt.Sprintf("%s/worker-test-%d", srv.URL, i), httprc.JSONTransformer[map[string]int64](), ) require.NoError(t, err, "worker stress test resource %d creation should succeed", i) require.NoError(t, ctrl.Add(ctx, resource), "adding worker stress test resource %d should succeed", i) } // Force refresh all resources simultaneously var wg sync.WaitGroup for i := range numResources { wg.Add(1) go func(i int) { defer wg.Done() url := fmt.Sprintf("%s/worker-test-%d", srv.URL, i) tctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if err := ctrl.Refresh(tctx, url); err != nil { t.Errorf("worker: should refresh resource %d: %v", i, err) } }(i) } wg.Wait() mu.Lock() finalCount := requestCount mu.Unlock() require.Greater(t, finalCount, int64(numResources), "should have processed multiple requests") }) t.Run("single worker processes requests sequentially", func(t *testing.T) { t.Parallel() // Reset counters mu.Lock() requestCount = 0 requestTimes = requestTimes[:0] mu.Unlock() traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } const numWorkers = 1 cl := httprc.NewClient( httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), httprc.WithWorkers(numWorkers), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Add multiple resources const numResources = 3 for i := range numResources { resource, err := httprc.NewResource[map[string]int64]( fmt.Sprintf("%s/sequential-test-%d", srv.URL, i), httprc.JSONTransformer[map[string]int64](), ) require.NoError(t, err, "sequential processing test resource %d creation should succeed", i) require.NoError(t, ctrl.Add(ctx, resource), "adding sequential processing test resource %d should succeed", i) } // Force refresh all resources var wg sync.WaitGroup for i := range numResources { wg.Add(1) go func(i int) { defer wg.Done() url := fmt.Sprintf("%s/sequential-test-%d", srv.URL, i) tctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() if err := ctrl.Refresh(tctx, url); err != nil { t.Errorf("sequential: should refresh resource %d: %v", i, err) } }(i) } wg.Wait() }) } func TestPeriodicRefresh(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) var requestCount int64 var mu sync.Mutex srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() requestCount++ count := requestCount mu.Unlock() w.Header().Set("Content-Type", "application/json") w.Header().Set("Cache-Control", "max-age=1") // Short cache for testing json.NewEncoder(w).Encode(map[string]int64{"count": count}) })) t.Cleanup(srv.Close) traceDst := io.Discard if testing.Verbose() { traceDst = os.Stderr } cl := httprc.NewClient( httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(traceDst, nil)))), ) ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("resource refreshes automatically", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]int64]( srv.URL+"/periodic-test", httprc.JSONTransformer[map[string]int64](), ) require.NoError(t, err) // Set short intervals for fast testing resource.SetMinInterval(10 * time.Millisecond) resource.SetMaxInterval(20 * time.Millisecond) require.NoError(t, ctrl.Add(ctx, resource), "should add resource to controller") // Get initial count var data1 map[string]int64 require.NoError(t, resource.Get(&data1), "getting initial data should succeed") initialCount := data1["count"] time.Sleep(5 * time.Second) // Get updated count var data2 map[string]int64 require.NoError(t, resource.Get(&data2), "getting updated data should succeed") newCount := data2["count"] require.Greater(t, newCount, initialCount, "resource should have been automatically refreshed") }) /* t.Run("multiple resources refresh independently", func(t *testing.T) { // Reset counter mu.Lock() requestCount = 0 mu.Unlock() const numResources = 3 resources := make([]httprc.Resource, numResources) for i := 0; i < numResources; i++ { resource, err := httprc.NewResource[map[string]int64]( fmt.Sprintf("%s/multi-periodic-%d", srv.URL, i), httprc.JSONTransformer[map[string]int64](), ) require.NoError(t, err) // Set different intervals for each resource interval := time.Duration(50*(i+1)) * time.Millisecond resource.SetMinInterval(interval) resource.SetMaxInterval(interval * 2) resources[i] = resource require.NoError(t, ctrl.Add(ctx, resource), "adding interval test resource %d should succeed", i) } // Wait for multiple refresh cycles time.Sleep(400 * time.Millisecond) // Each resource should have been refreshed at least once for i, resource := range resources { var data map[string]int64 err := resource.Get(&data) require.NoError(t, err, "resource %d should be accessible", i) require.Greater(t, data["count"], int64(0), "resource %d should have been refreshed", i) } mu.Lock() finalRequestCount := requestCount mu.Unlock() require.Greater(t, finalRequestCount, int64(numResources), "should have made multiple requests due to refreshes") }) */ } func TestEdgeCases(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) t.Run("empty response body", func(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { // Return empty body w.WriteHeader(http.StatusOK) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[[]byte]( srv.URL, httprc.BytesTransformer(), ) require.NoError(t, err, "empty response test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding empty response test resource should succeed") var data []byte require.NoError(t, resource.Get(&data), "getting empty data should succeed") require.Empty(t, data) }) t.Run("very large response", func(t *testing.T) { t.Parallel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { // Generate a large response data := make(map[string]string) for i := range 10000 { data[fmt.Sprintf("key_%d", i)] = fmt.Sprintf("value_%d_with_lots_of_data_to_make_it_large", i) } json.NewEncoder(w).Encode(data) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) resource, err := httprc.NewResource[map[string]string]( srv.URL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "large response test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding large response test resource should succeed") var data map[string]string require.NoError(t, resource.Get(&data), "getting large response data should succeed") require.Len(t, data, 10000) }) t.Run("rapid add and remove", func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Rapidly add and remove resources for i := range 100 { testURL := fmt.Sprintf("%s/rapid-test-%d", srv.URL, i) resource, err := httprc.NewResource[map[string]string]( testURL, httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "rapid add/remove test resource %d creation should succeed", i) require.NoError(t, ctrl.Add(ctx, resource), "adding rapid add/remove test resource %d should succeed", i) // Immediately remove it require.NoError(t, ctrl.Remove(ctx, testURL), "removing rapid add/remove test resource %d should succeed", i) } }) t.Run("invalid JSON with fallback", func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.Write([]byte("invalid json {")) })) defer srv.Close() cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) // Use bytes transformer instead of JSON for invalid JSON resource, err := httprc.NewResource[[]byte]( srv.URL, httprc.BytesTransformer(), ) require.NoError(t, err, "invalid JSON fallback test resource creation should succeed") require.NoError(t, ctrl.Add(ctx, resource), "adding invalid JSON fallback test resource should succeed") var data []byte require.NoError(t, resource.Get(&data), "getting invalid JSON fallback data should succeed") require.Equal(t, []byte("invalid json {"), data) }) } func TestResourceLifecycle(t *testing.T) { t.Parallel() ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) var requestPhases []string var mu sync.Mutex srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { mu.Lock() requestPhases = append(requestPhases, "request_received") mu.Unlock() time.Sleep(10 * time.Millisecond) // Simulate processing json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) mu.Lock() requestPhases = append(requestPhases, "response_sent") mu.Unlock() })) t.Cleanup(srv.Close) cl := httprc.NewClient() ctrl, err := cl.Start(ctx) require.NoError(t, err) t.Cleanup(func() { ctrl.Shutdown(time.Second) }) t.Run("full lifecycle", func(t *testing.T) { t.Parallel() resource, err := httprc.NewResource[map[string]string]( srv.URL+"/lifecycle-test", httprc.JSONTransformer[map[string]string](), ) require.NoError(t, err, "lifecycle test resource creation should succeed") // Initially, resource should not be ready require.False(t, resource.IsBusy()) // Add resource (this will trigger the first fetch) require.NoError(t, ctrl.Add(ctx, resource), "adding lifecycle test resource should succeed") // Resource should now be available var data map[string]string require.NoError(t, resource.Get(&data), "getting lifecycle test data should succeed") require.Equal(t, "ok", data["status"]) // Lookup should find the resource found, err := ctrl.Lookup(ctx, srv.URL+"/lifecycle-test") require.NoError(t, err) require.Equal(t, resource.URL(), found.URL()) // Refresh should work require.NoError(t, ctrl.Refresh(ctx, srv.URL+"/lifecycle-test"), "refreshing lifecycle test resource should succeed") // Remove should work require.NoError(t, ctrl.Remove(ctx, srv.URL+"/lifecycle-test"), "removing lifecycle test resource should succeed") // Lookup should now fail _, err = ctrl.Lookup(ctx, srv.URL+"/lifecycle-test") require.Error(t, err) mu.Lock() phases := make([]string, len(requestPhases)) copy(phases, requestPhases) mu.Unlock() require.NotEmpty(t, phases, "should have recorded request phases") }) }