pax_global_header00006660000000000000000000000064152370212670014517gustar00rootroot0000000000000052 comment=9d078edadc935c3d202915a71b12bd5492051038 OpenVoxProject-trapperkeeper-comidi-metrics-0545740/000077500000000000000000000000001523702126700224135ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/000077500000000000000000000000001523702126700237535ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/dependabot.yml000066400000000000000000000005751523702126700266120ustar00rootroot00000000000000version: 2 updates: # raise PRs for gem updates - package-ecosystem: bundler directory: "/" schedule: interval: daily time: "13:00" open-pull-requests-limit: 10 # Maintain dependencies for GitHub Actions - package-ecosystem: github-actions directory: "/" schedule: interval: daily time: "13:00" open-pull-requests-limit: 10 OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/release.yml000066400000000000000000000013231523702126700261150ustar00rootroot00000000000000--- # https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes changelog: exclude: labels: - duplicate - invalid - modulesync - question - skip-changelog - wont-fix - wontfix categories: - title: Breaking Changes 🛠 labels: - backwards-incompatible - title: New Features 🎉 labels: - enhancement - title: Bug Fixes 🐛 labels: - bug - title: Documentation Updates 📚 labels: - documentation - docs - title: Dependency Updates ⬆️ labels: - dependencies - title: Other Changes labels: - "*" OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/workflows/000077500000000000000000000000001523702126700260105ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/workflows/pr-testing.yaml000066400000000000000000000027341523702126700307760ustar00rootroot00000000000000--- name: lein_test on: pull_request: {} push: branches: - main # minimal permissions permissions: contents: read jobs: run-lein-tests: name: lein test - Java ${{ matrix.java }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: java: - '25' - '21' - '17' steps: - name: Check out repository code uses: actions/checkout@v7 - name: Setup java uses: actions/setup-java@v5 with: distribution: temurin java-version: ${{ matrix.java }} - name: Install Clojure tools uses: DeLaGuardo/setup-clojure@13.5 with: cli: latest # Clojure CLI based on tools.deps lein: latest # Leiningen boot: latest # Boot.clj bb: latest # Babashka clj-kondo: latest # Clj-kondo cljstyle: latest # cljstyle zprint: latest # zprint - name: Run lein tests with update profile run: | set -x set -e lein version echo "Running tests" lein -U test tests: if: always() needs: - run-lein-tests runs-on: ubuntu-24.04 name: Test suite steps: - name: Decide whether the needed jobs succeeded or failed uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.github/workflows/release.yaml000066400000000000000000000007411523702126700303160ustar00rootroot00000000000000--- name: Release and publish package to clojars.org permissions: contents: write packages: write on: workflow_dispatch: jobs: release: uses: 'openvoxproject/shared-actions/.github/workflows/clojure_release.yml@main' secrets: github_pat: ${{ secrets.OPENVOXBOT_COMMIT_AND_PRS }} ssh_private_key: ${{ secrets.OPENVOXBOT_SSH_PRIVATE_KEY }} clojars_username: ${{ secrets.CLOJARS_USERNAME }} clojars_password: ${{ secrets.CLOJARS_PASSWORD }} OpenVoxProject-trapperkeeper-comidi-metrics-0545740/.gitignore000066400000000000000000000002121523702126700243760ustar00rootroot00000000000000pom.xml pom.xml.asc *jar /lib/ /classes/ /target/ /checkouts/ .lein-deps-sum .lein-repl-history .lein-plugins/ .lein-failures .nrepl-port OpenVoxProject-trapperkeeper-comidi-metrics-0545740/LICENSE000066400000000000000000000012051523702126700234160ustar00rootroot00000000000000trapperkeeper-comidi-metrics Copyright (C) 2015 Puppet Labs Inc Puppet Labs can be contacted at: info@puppetlabs.com Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. OpenVoxProject-trapperkeeper-comidi-metrics-0545740/README.md000066400000000000000000000126071523702126700237000ustar00rootroot00000000000000## `trapperkeeper-comidi-metrics` [![Clojars Project](http://clojars.org/org.openvoxproject/trapperkeeper-comidi-metrics/latest-version.svg)](http://clojars.org/org.openvoxproject/trapperkeeper-comidi-metrics) `trapperkeeper-comidi-metrics` is a library that provides Ring middleware to automatically track metrics for requests to all of your [`comidi`](https://github.com) HTTP routes. To get the most value out of this library, use it in concert with [trapperkeeper-metrics](https://github.com/openvoxproject/trapperkeeper-metrics) (to take advantage of the built-in ring-middleware for tracking HTTP metrics) and the [Trapperkeeper Status Service](https://github.com/openvoxproject/trapperkeeper-status) (to expose the most useful metrics data from your app via HTTP). ### Quick Start This repo includes an example app that is intended to show how to tie all of those pieces together. The easiest way to get started is probably to run the example app, and then take a peek into the code. To start the example web app, run: `lein example` Then, in another shell, run: `lein example-data` This will generate some random requests to the web app, and then issue a request to the status endpoint to print some summary data about the metrics. For more info about the HTTP metrics, see the [comidi metrics documentation](./documentation/metrics.md). ### Detailed Info `trapperkeeper-comidi-metrics` contains a Ring middleware that can be used in concert with a `MetricRegistry` (from the [dropwizard Metrics library](https://dropwizard.github.io/metrics)) to automatically track metrics for all of your `comidi` routes. This can be used in conjunction with the [Trapperkeeper Status Service](https://github.com/openvoxproject/trapperkeeper-status) to easily expose debugging / telemetry information via HTTP. Here's an example of how to use the Ring middleware: ```clj (defn my-routes [url-prefix] (comidi/context path (comidi/context "/v1" (comidi/routes (comidi/GET ["/foo"] request (handle-foo request)) (comidi/ANY ["/bar/" :bar] [bar] (handle-bar bar)))))) (tk/defservice my-web-service [[:WebroutingService add-ring-handler get-route] [:MetricsService get-metrics-registry]] (init [this context] (let [path (get-route this) metrics-registry (get-metrics-registry) routes (my-routes path) route-metadata (comidi/route-metadata routes) http-metrics (http-metrics/initialize-http-metrics! metrics-registry "localhost" route-metadata) handler (-> (comidi/routes->handler routes) (http-metrics/wrap-with-request-metrics http-metrics) (comidi/wrap-with-route-metadata routes))] (add-ring-handler this handler) (log/info "REQUEST SUMMARY:" (http-metrics/request-summary http-metrics))) context)) ``` (Note that `http-metrics/wrap-with-request-metrics` relies on data from the `comidi/wrap-with-route-metadata` middleware.) In the example above, the call to `request-summary` will return counts of zero for all of the routes, because no requests have been handled yet, but the data structure returned by `request-summary` will look something like this in a real-world scenario: ``` {:routes {:total {:route-id "total", :count 100, :mean 18, :aggregate 1800}, :other {:route-id "other", :count 0, :mean 0, :aggregate 0}, "example-v1-foo" {:route-id "example-v1-foo", :count 34, :mean 10, :aggregate 340}, "example-v1-bar-:bar" {:route-id "example-v1-bar-:bar", :count 30, :mean 18, :aggregate 540}}, :sorted-routes [{:route-id "total", :count 100, :mean 18, :aggregate 1800}, {:route-id "example-v1-bar-:bar", :count 30, :mean 18, :aggregate 540}, {:route-id "example-v1-foo", :count 34, :mean 10, :aggregate 340}, {:route-id "other", :count 0, :mean 0, :aggregate 0}]} ``` For each route in your `comidi` route tree, `initialize-http-metrics!` will generate a unique `route-id`. Then, `request-summary` can be used to retrieve the metrics data for each route. The return value of `request-summary` is a map with two keys; `:routes`, and `:sorted-routes`. They contain the same data, but `:routes` contains a nested map whose keys are the `route-ids`, so that you can look up the data for a specific route, while the value `:sorted-routes` is a vector sorted by the aggregate amount of time spent handling requests for each route (in descending order). For each route, the data returned includes a `:count` of how many requests have been made, a `:mean` time (in milliseconds) indicating how long it's taken to handle the average request, and an `:aggregate` time (in milliseconds) showing how much total time has been spent handling requests for that route. There are two special route ids included: `:total`, which summarizes all requests across all routes, and `:other`, which tracks metrics for requests that did not match any of your routes. For a complete example that illustrates how to expose the HTTP metrics data via the [Trapperkeeper Status Service](https://github.com/openvoxproject/trapperkeeper-status), see the [source code for the sample web application](../dev/example/comidi_metrics_web_app.clj). ## License See [LICENSE](LICENSE). ## Support GitHub issues and PRs are welcome! Additionally, drop us a line in [the Vox Pupuli Slack](https://voxpupuli.org/connect/). OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev-resources/000077500000000000000000000000001523702126700252015ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev-resources/logback-test.xml000066400000000000000000000005401523702126700303010ustar00rootroot00000000000000 %d %-5p [%c{2}] %m%n OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/000077500000000000000000000000001523702126700231715ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/000077500000000000000000000000001523702126700246245ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/bootstrap.cfg000066400000000000000000000006571523702126700273320ustar00rootroot00000000000000example.comidi-metrics-web-app/example-web-service puppetlabs.trapperkeeper.services.metrics.metrics-service/metrics-service puppetlabs.trapperkeeper.services.status.status-service/status-service puppetlabs.trapperkeeper.services.webserver.jetty-service/jetty-service puppetlabs.trapperkeeper.services.webrouting.webrouting-service/webrouting-service puppetlabs.trapperkeeper.services.scheduler.scheduler-service/scheduler-service OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/comidi_metrics_web_app.clj000066400000000000000000000115741523702126700320150ustar00rootroot00000000000000(ns example.comidi-metrics-web-app (:require [clojure.tools.logging :as log] [puppetlabs.trapperkeeper.core :as tk] [puppetlabs.comidi :as comidi] [puppetlabs.trapperkeeper.services.status.status-core :as status] [puppetlabs.metrics.http :as http-metrics] [puppetlabs.metrics :as metrics]) (:import (java.util.concurrent TimeUnit))) ;; TODO: it might be worth moving this example app out into its own repo at ;; some point, because eventually it will also demo our comidi->swagger ;; integration, etc. Or we could put it into comidi itself, if we OSS this ;; metrics library. (defn rand-sleep [min max] (Thread/sleep (+ min (rand-int max)))) (defn handle-foo [req] (log/info "Handling foo request") (rand-sleep 5 10) "foo!") (defn handle-bar [bar] (log/info "Handling bar request, params:" bar) (rand-sleep 10 20) "bar!") (defn baz-task1 [timer] (metrics/time! timer (rand-sleep 5 10))) (defn baz-task2 [timer] (metrics/time! timer (rand-sleep 20 50))) (defn handle-baz [app-metrics baz bam] (log/info "Handling baz request, params:" baz bam) (baz-task1 (:baz-task1-timer app-metrics)) (if (= (rand-int 3) 0) (baz-task2 (:baz-task2-timer app-metrics))) "baz!") (defn example-routes [path app-metrics] ;; NOTE: the HTTP metrics are automatically built up from your comidi routes. ;; The example traffic generator is also dynamic based on these routes. ;; ;; Try adding or removing some routes from this route tree and re-running ;; the app, and notice that the metrics will be updated accordingly! (comidi/context path (comidi/context "/v1" (comidi/routes (comidi/ANY ["/foo"] request (handle-foo request)) (comidi/ANY ["/bar/" :bar] [bar] (handle-bar bar)) (comidi/ANY ["/baz/" :baz "/bam/" :bam] [baz bam] (handle-baz app-metrics baz bam)))))) (defn build-handler [routes http-metrics] (-> (comidi/routes->handler routes) (http-metrics/wrap-with-request-metrics http-metrics) (comidi/wrap-with-route-metadata routes))) (defn register-app-metrics [metrics-registry] (log/info "Registering app-specific metrics for Example Web Service") {:baz-task1-timer (.timer metrics-registry (metrics/host-metric-name "localhost" "baz-task1-timer")) :baz-task2-timer (.timer metrics-registry (metrics/host-metric-name "localhost" "baz-task2-timer"))}) (def status-version 1) (defn create-status-callback-fn [http-metrics app-metrics] (letfn [(get-count [id] (.. (get app-metrics id) getCount)) (get-mean [id] (->> (get app-metrics id) .getSnapshot .getMean (.toMillis TimeUnit/NANOSECONDS)))] (fn [level] (let [level>= (partial status/compare-levels >= level) status (cond-> {} ;; no extra status info to add for info level, just including ;; this for the sake of example (level>= :info) identity (level>= :debug) (-> (assoc-in [:debug :route-metrics] (http-metrics/request-summary http-metrics)) (assoc-in [:debug :app-metrics] {:baz-task1 {:count (get-count :baz-task1-timer) :mean (get-mean :baz-task1-timer)} :baz-task2 {:count (get-count :baz-task2-timer) :mean (get-mean :baz-task2-timer)}})))] {:state :running :status status})))) (tk/defservice example-web-service [[:WebroutingService add-ring-handler get-route] [:MetricsService get-metrics-registry] [:StatusService register-status]] (init [this context] (let [path (get-route this) metrics-registry (get-metrics-registry) app-metrics (register-app-metrics metrics-registry) routes (example-routes path app-metrics) route-metadata (comidi/route-metadata routes) http-metrics (http-metrics/initialize-http-metrics! metrics-registry "localhost" route-metadata) handler (build-handler routes http-metrics)] (log/infof "Registering status callback for Example Web Service") (register-status "example-web-service" (status/get-artifact-version "org.openvoxproject" "trapperkeeper-metrics") status-version (create-status-callback-fn http-metrics app-metrics)) (log/infof "Registering Example Web Service at '%s'" path) (add-ring-handler this handler)) context)) (defn -main [& args] (tk/boot-with-cli-data {:config "./dev/example/example.conf" :bootstrap-config "./dev/example/bootstrap.cfg"})) OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/example.conf000066400000000000000000000005161523702126700271300ustar00rootroot00000000000000global: { logging-config: "./dev/example/logback.xml" } webserver: { host: localhost port: 8080 } web-router-service: { "puppetlabs.trapperkeeper.services.status.status-service/status-service": "/status" "example.comidi-metrics-web-app/example-web-service": "/example" } metrics: { server-id: "localhost" } OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/logback.xml000066400000000000000000000005371523702126700267550ustar00rootroot00000000000000 %d %-5p [%c{2}] %m%n OpenVoxProject-trapperkeeper-comidi-metrics-0545740/dev/example/traffic_generator.clj000066400000000000000000000044551523702126700310120ustar00rootroot00000000000000(ns example.traffic-generator (:require [puppetlabs.http.client.sync :as http-client] [puppetlabs.comidi :as comidi] [example.comidi-metrics-web-app :as example-app] [clojure.string :as str] [cheshire.core :as json] [clojure.pprint :as pprint])) (defn http-get [uri] (http-client/get uri {:as :text})) (defn random-string [] (rand-nth ["the" "quick" "brown" "fox" "jumps" "over" "the" "lazy" "dog"])) (defn path-segment-to-url-segment [path-segment] (cond (string? path-segment) (str/replace path-segment #"^/?([^/]+)/?" "$1") (keyword? path-segment) (random-string) :else (throw (IllegalStateException. (str "Unrecognized path segment:" path-segment))))) (defn path-to-url [path] (str/join "/" (map path-segment-to-url-segment (filter (some-fn keyword? not-empty) path)))) (defn generate-random-uri [route-metadata] (let [route (rand-nth (:routes route-metadata))] (str "http://localhost:8080/" (path-to-url (:path route))))) (defn generate-random-traffic [num-requests] ;; NOTE: this is a little crude. We are hard-coding the URL prefix rather than ;; getting it from the web routing config, and we are also assuming a fairly ;; homogenous route structure where all of the routes will accept GET requests ;; with no query parameters. ;; ;; That said, there is some promise here for being able to do generative testing ;; by writing generators based on comidi route metadata in the future! Especially ;; when we add some kind of prismatic-schema -> ring-swagger functionality to ;; comidi; then we might be able to use the schema data to do some fairly ;; sophisticated things with test generators. (let [route-metadata (comidi/route-metadata (example-app/example-routes "/example" {}))] (dotimes [i num-requests] (let [random-uri (generate-random-uri route-metadata)] (http-get random-uri))))) (defn -main [& args] (println "Generating 100 random requests.") (generate-random-traffic 100) (let [status-url "http://localhost:8080/status/v1/services?level=debug"] (println "Requesting status info from" status-url) (let [resp (http-get status-url)] (pprint/pprint (json/parse-string (:body resp))))))OpenVoxProject-trapperkeeper-comidi-metrics-0545740/project.clj000066400000000000000000000065161523702126700245630ustar00rootroot00000000000000(defproject org.openvoxproject/trapperkeeper-comidi-metrics "1.1.1" :description "Comidi/HTTP Metrics for Trapperkeeper" :url "http://github.com/openvoxproject/trapperkeeper-comidi-metrics" :license {:name "Apache-2.0" :url "https://www.apache.org/licenses/LICENSE-2.0.txt"} :min-lein-version "2.12.0" :pedantic? :abort ;; Generally, try to keep version pins in :managed-dependencies and the libraries ;; this project actually uses in :dependencies, inheriting the version from ;; :managed-dependencies. This prevents endless version conflicts due to deps of deps. ;; Renovate should keep the versions largely in sync between projects. :managed-dependencies [[org.clojure/clojure "1.12.5"] [org.clojure/java.jmx "1.1.1"] [org.clojure/tools.logging "1.3.1"] [org.clojure/tools.macro "0.2.2"] [cheshire "6.2.0"] [clj-time "0.15.2"] [commons-codec "1.22.1"] [commons-io "2.22.0"] [io.dropwizard.metrics/metrics-core "3.2.6"] [org.bouncycastle/bcpkix-jdk18on "1.84"] [org.openvoxproject/comidi "1.1.4"] [org.openvoxproject/http-client "2.4.1"] [org.openvoxproject/i18n "1.0.5"] [org.openvoxproject/kitchensink "3.5.8"] [org.openvoxproject/kitchensink "3.5.8" :classifier "test"] [org.openvoxproject/trapperkeeper "5.0.5"] [org.openvoxproject/trapperkeeper "5.0.5" :classifier "test"] [org.openvoxproject/trapperkeeper-metrics "2.3.2"] [org.openvoxproject/trapperkeeper-status "1.5.1"] [org.openvoxproject/trapperkeeper-webserver "12.1.1"] [org.ring-clojure/ring-core-protocols "1.15.5"] [org.ring-clojure/ring-websocket-protocols "1.15.5"] [org.slf4j/slf4j-api "2.0.18"] [prismatic/schema "1.4.2"] [ring/ring-codec "1.3.0"] [ring/ring-core "1.15.5"]] :dependencies [[org.clojure/clojure] [org.openvoxproject/comidi] [org.openvoxproject/trapperkeeper-metrics] [prismatic/schema]] :deploy-repositories [["releases" {:url "https://clojars.org/repo" :username :env/CLOJARS_USERNAME :password :env/CLOJARS_PASSWORD :sign-releases false}]] :profiles {:dev {:source-paths ["dev"] :dependencies [[org.bouncycastle/bcpkix-jdk18on] [org.openvoxproject/http-client] [org.openvoxproject/kitchensink :classifier "test"] [org.openvoxproject/trapperkeeper :classifier "test"] [org.openvoxproject/trapperkeeper-status] [org.openvoxproject/trapperkeeper-webserver]]}} :aliases {"example" ["run" "-m" "example.comidi-metrics-web-app"] "example-data" ["run" "-m" "example.traffic-generator"]}) OpenVoxProject-trapperkeeper-comidi-metrics-0545740/renovate.json000066400000000000000000000004061523702126700251310ustar00rootroot00000000000000{ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "addLabels": ["renovate", "dependencies"], "assigneesFromCodeOwners": true, "automerge": true, "automergeType": "pr", "dependencyDashboard": true, "enabledManagers": ["leiningen"] } OpenVoxProject-trapperkeeper-comidi-metrics-0545740/src/000077500000000000000000000000001523702126700232025ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/src/puppetlabs/000077500000000000000000000000001523702126700253615ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/src/puppetlabs/metrics/000077500000000000000000000000001523702126700270275ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/src/puppetlabs/metrics/http.clj000066400000000000000000000153301523702126700305020ustar00rootroot00000000000000;; Utilities for tracking metrics on http requests. (ns puppetlabs.metrics.http (:import (java.util.concurrent TimeUnit) (com.codahale.metrics Timer Counter Histogram MetricRegistry)) (:require [schema.core :as schema] [puppetlabs.comidi :as comidi] [puppetlabs.metrics :as metrics])) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Schemas (def TimersMap {:other Timer schema/Str Timer}) (def HttpMetrics {:active-counter Counter :active-histo Histogram :total-timer Timer :route-timers TimersMap}) (def ReservedRouteIdentifier (schema/enum :total :other)) (def RouteIdentifier (schema/conditional keyword? ReservedRouteIdentifier string? schema/Str)) (def RouteSummary {:route-id RouteIdentifier :count schema/Int :mean schema/Num :aggregate schema/Num}) (def RequestSummary {:routes {:other RouteSummary :total RouteSummary schema/Str RouteSummary} :sorted-routes [RouteSummary]}) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Private (schema/defn register-metrics-for-endpoint :- TimersMap "Initialize metrics for an http endpoint and add them to the registry. This includes a Timer for the endpoint itself, and a Ratio to track the percentage of total requests that were directed to this endpoint." [registry :- MetricRegistry total-requests :- Timer metric-name-fn :- (schema/pred ifn?) acc :- TimersMap endpoint :- schema/Str] (if (contains? acc endpoint) acc (let [timer (.timer registry (metric-name-fn (str endpoint "-requests")))] (.register registry (metric-name-fn (str endpoint "-percentage")) (metrics/metered-ratio timer total-requests)) (assoc acc endpoint timer)))) (schema/defn register-http-metrics :- TimersMap "Initialize metrics for a list of http endpoints." [registry :- MetricRegistry total-requests :- Timer metric-name-fn :- (schema/pred ifn?) route-names :- [schema/Str]] (let [other-timer (.timer registry (metric-name-fn "other-requests"))] (.register registry (metric-name-fn "other-percentage") (metrics/metered-ratio other-timer total-requests)) (reduce (partial register-metrics-for-endpoint registry total-requests metric-name-fn) {:other other-timer} route-names))) (defn find-http-route-timer "Given a route-id and a map of timers, return the timer for the requested route, or the catch-all `:other` timer if there is no timer for the route." [route-id route-timers] (if-let [timer (route-timers route-id)] timer (:other route-timers))) (schema/defn ^:always-validate assoc-route-summary :- {RouteIdentifier RouteSummary} "Add summary information for the given route-id to the accumulator map." [acc :- {RouteIdentifier RouteSummary} route-id :- RouteIdentifier route-timer :- Timer] (let [count (.getCount route-timer) mean (->> route-timer .getSnapshot .getMean (.toMillis TimeUnit/NANOSECONDS))] (assoc acc route-id {:route-id route-id :count count :mean mean :aggregate (* mean count)}))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;; Public (schema/defn ^:always-validate initialize-http-metrics! :- (schema/maybe HttpMetrics) "Initialize a MetricRegistry with metrics for a list of HTTP endpoints. The registry will be populated with a `num-cpus` metric for the system, a Counter and a Histogram for tracking active requests, and a Timer that will be used to measure all requests. `route-metadata` is the comidi route metadata for all of the routes that we want to track metrics for; for each of these, a Timer will be initialized, as well as a Ratio that keeps track of the percentage of total requests that were directed to the named endpoint. This function is intended for use with the `wrap-with-request-metrics` Ring middleware from this library, and the `wrap-with-route-metadata` Ring middleware from comidi." [registry :- (schema/maybe MetricRegistry) hostname :- schema/Str route-metadata :- comidi/RouteMetadata] (when registry (metrics/register registry (metrics/host-metric-name hostname "num-cpus") (metrics/gauge (.availableProcessors (Runtime/getRuntime)))) (let [active-counter (.counter registry (metrics/http-metric-name hostname "active-requests")) active-histo (.histogram registry (metrics/http-metric-name hostname "active-histo")) total-timer (.timer registry (metrics/http-metric-name hostname "total-requests")) route-timers (register-http-metrics registry total-timer (partial metrics/http-metric-name hostname) (map :route-id (:routes route-metadata)))] {:active-counter active-counter :active-histo active-histo :total-timer total-timer :route-timers route-timers}))) (schema/defn ^:always-validate wrap-with-request-metrics :- (schema/pred ifn?) "Ring middleware. Wraps the given ring handler with code that will update the various metrics created by a call to `initialize-http-metrics!`, based on whether or not the request is directed to one of the endpoints that metrics are being tracked for. The comidi route metadata (via the comidi `wrap-with-route-metadata` Ring middleware) will be used to determine which metric should be associated with the request." [app :- (schema/pred ifn?) {:keys [active-counter active-histo total-timer route-timers] :as http-metrics} :- (schema/maybe HttpMetrics)] (if-not http-metrics app (fn [req] (.inc active-counter) (.update active-histo (.getCount active-counter)) (try (metrics/time! total-timer (let [resp (if-let [timer (find-http-route-timer (get-in req [:route-info :route-id]) route-timers)] (metrics/time! timer (app req)) (app req))] resp)) (finally (.dec active-counter) (.update active-histo (.getCount active-counter))))))) (schema/defn ^:always-validate request-summary :- RequestSummary "Build a summary of request data to all of the routes registered in the metrics." [metrics :- HttpMetrics] (let [route-summaries (-> (reduce-kv assoc-route-summary {} (:route-timers metrics)) (assoc-route-summary :total (:total-timer metrics)))] {:routes route-summaries :sorted-routes (sort-by :aggregate > (vals route-summaries))})) OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/000077500000000000000000000000001523702126700233725ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/example/000077500000000000000000000000001523702126700250255ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/example/comidi_metrics_web_app_test.clj000066400000000000000000000071331523702126700332510ustar00rootroot00000000000000(ns example.comidi-metrics-web-app-test (:require [clojure.test :refer :all] [puppetlabs.trapperkeeper.bootstrap :as tk-bootstrap] [puppetlabs.trapperkeeper.config :as tk-config] [puppetlabs.trapperkeeper.testutils.bootstrap :refer [with-app-with-config]] [puppetlabs.http.client.sync :as http-client] [cheshire.core :as json] [example.traffic-generator :as traffic-generator] [puppetlabs.trapperkeeper.testutils.logging :as logutils])) (defn get-services-from-bootstrap [] (tk-bootstrap/parse-bootstrap-config! "./dev/example/bootstrap.cfg")) (defn get-config-from-file [] (tk-config/load-config "./dev/example/example.conf")) (defn http-get [uri] (http-client/get uri {:as :text})) (defn parse [resp] (-> resp :body json/parse-string)) (deftest test-example-web-service (logutils/with-test-logging (with-app-with-config app (get-services-from-bootstrap) (assoc-in (get-config-from-file) [:global :logging-config] "./dev-resources/logback-test.xml") (traffic-generator/generate-random-traffic 49) (testing "example app endpoints are accessible" (let [resp (http-get "http://localhost:8080/example/v1/foo")] (is (= 200 (:status resp))))) (testing "status endpoint is accessible" (let [resp (http-get "http://localhost:8080/status/v1/services")] (is (= 200 (:status resp))) (let [body (parse resp)] (is (contains? body "example-web-service")) (is (empty? (get-in body ["example-web-service" "status"])))))) (testing "at debug level, status endpoint returns metrics data" (let [resp (http-get "http://localhost:8080/status/v1/services?level=debug")] (is (= 200 (:status resp))) (let [body (parse resp)] (is (contains? body "example-web-service")) (let [route-metrics (get-in body ["example-web-service" "status" "debug" "route-metrics" "routes"])] (is (= #{"total" "other" "example-v1-foo" "example-v1-bar-:bar" "example-v1-baz-:baz-bam-:bam"} (->> route-metrics keys set))) (is (= 50 (get-in route-metrics ["total" "count"]))) (is (= 0 (get-in route-metrics ["other" "count"]))) (is (= 50 (apply + (map #(get-in route-metrics [% "count"]) ["example-v1-foo" "example-v1-bar-:bar" "example-v1-baz-:baz-bam-:bam"])))) (if (= 0 (get-in route-metrics ["example-v1-foo" "count"])) (is (= 0 (get-in route-metrics ["example-v1-foo" "mean"]))) (is (<= 5 (get-in route-metrics ["example-v1-foo" "mean"])))) (if (= 0 (get-in route-metrics ["example-v1-bar-:bar" "count"])) (is (= 0 (get-in route-metrics ["example-v1-bar-:bar" "mean"]))) (is (<= 10 (get-in route-metrics ["example-v1-bar-:bar" "mean"]))))) (let [app-metrics (get-in body ["example-web-service" "status" "debug" "app-metrics"])] (is (= #{"baz-task1" "baz-task2"} (->> app-metrics keys set))) (if (= 0 (get-in app-metrics ["baz-task1" "count"])) (is (= 0 (get-in app-metrics ["baz-task1" "mean"]))) (is (<= 5 (get-in app-metrics ["baz-task1" "mean"])))) (if (= 0 (get-in app-metrics ["baz-task2" "count"])) (is (= 0 (get-in app-metrics ["baz-task2" "mean"]))) (is (<= 20 (get-in app-metrics ["baz-task2" "mean"]))))))))))) OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/puppetlabs/000077500000000000000000000000001523702126700255515ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/puppetlabs/metrics/000077500000000000000000000000001523702126700272175ustar00rootroot00000000000000OpenVoxProject-trapperkeeper-comidi-metrics-0545740/test/puppetlabs/metrics/http_test.clj000066400000000000000000000136051523702126700317340ustar00rootroot00000000000000(ns puppetlabs.metrics.http-test (:import (com.codahale.metrics MetricRegistry RatioGauge Timer Gauge)) (:require [clojure.test :refer :all] [puppetlabs.metrics.http :refer :all] [schema.test :as schema-test] [puppetlabs.comidi :as comidi])) (use-fixtures :once schema-test/validate-schemas) (deftest test-initialize-http-metrics! (testing "initialize-http-metrics! should create metrics for all specified HTTP endpoints" (let [routes (comidi/routes (comidi/GET ["/foo/something"] request "foo!") (comidi/POST ["/bar" :bar] request "bar!") (comidi/PUT ["/baz/" :baz "/bam/" :bam] request "baz!")) route-meta (comidi/route-metadata routes) registry (MetricRegistry.) http-metrics (initialize-http-metrics! registry "localhost" route-meta) metrics-map (.getMetrics registry) num-cpus (.get metrics-map "puppetlabs.localhost.num-cpus")] (is (= #{"foo-something" "bar-:bar" "baz-:baz-bam-:bam" :other} (-> http-metrics :route-timers keys set))) (is (instance? Gauge num-cpus)) (is (= (.availableProcessors (Runtime/getRuntime)) (.getValue num-cpus))) (is (instance? Timer (.get metrics-map "puppetlabs.localhost.http.other-requests"))) (is (instance? RatioGauge (.get metrics-map "puppetlabs.localhost.http.other-percentage"))) (is (instance? RatioGauge (.get metrics-map "puppetlabs.localhost.http.foo-something-percentage"))) (is (instance? RatioGauge (.get metrics-map "puppetlabs.localhost.http.bar-:bar-percentage"))) (is (instance? RatioGauge (.get metrics-map "puppetlabs.localhost.http.baz-:baz-bam-:bam-percentage")))))) (deftest test-routes-with-same-name (testing "should re-use metrics objects for routes that only differ by HTTP verb" ;; TODO: we should consider whether or not this is the behavior we want. It ;; might be useful to have different metrics per-verb, but then our naming ;; strategy will get even uglier. (let [routes (comidi/routes (comidi/GET ["/foo" :foo] request "GET foo!") (comidi/PUT ["/foo" :foo] request "PUT foo!")) route-meta (comidi/route-metadata routes) registry (MetricRegistry.) http-metrics (initialize-http-metrics! registry "localhost" route-meta)] (is (= #{"foo-:foo" :other} (-> http-metrics :route-timers keys set)))))) (deftest test-wrap-with-request-metrics (let [registry (MetricRegistry.) routes (comidi/routes (comidi/ANY ["/foo/" :foo] request "foo!")) route-meta (comidi/route-metadata routes) http-metrics (initialize-http-metrics! registry "localhost" route-meta) metrics-map (.getMetrics registry) ring-app (-> (comidi/routes->handler routes) (wrap-with-request-metrics http-metrics) (comidi/wrap-with-route-metadata routes)) total-timer (:total-timer http-metrics) foo-timer (get-in http-metrics [:route-timers "foo-:foo"]) other-timer (get-in http-metrics [:route-timers :other]) foo-ratio (.get metrics-map "puppetlabs.localhost.http.foo-:foo-percentage") other-ratio (.get metrics-map "puppetlabs.localhost.http.other-percentage")] (testing "increments endpoint timer when route matches" (ring-app {:uri "/foo/something"}) (is (= 0 (.getCount other-timer))) (is (= 1 (.getCount foo-timer))) (is (= 1 (.getCount total-timer))) (is (= 1.0 (.. foo-ratio getRatio getValue))) (is (= 0.0 (.. other-ratio getRatio getValue)))) (testing "increments :other timer when route doesn't match route-metadata" (ring-app {:uri "/bar/blah"}) (is (= 1 (.getCount other-timer))) (is (= 1 (.getCount foo-timer))) (is (= 2 (.getCount total-timer))) (is (= 0.5 (.. foo-ratio getRatio getValue))) (is (= 0.5 (.. other-ratio getRatio getValue)))))) (deftest test-request-summary (testing "request summary returns a list sorted by aggregate time spent in requests" (let [registry (MetricRegistry.) routes (comidi/routes (comidi/ANY ["/foo/" :foo] request (do (Thread/sleep 5) "foo!")) (comidi/ANY ["/bar/" :bar] request (do (Thread/sleep 20) "bar!"))) route-meta (comidi/route-metadata routes) http-metrics (initialize-http-metrics! registry "localhost" route-meta) ring-app (-> (comidi/routes->handler routes) (wrap-with-request-metrics http-metrics) (comidi/wrap-with-route-metadata routes))] (ring-app {:uri "/foo/something"}) (ring-app {:uri "/foo/something"}) (ring-app {:uri "/bar/something"}) (ring-app {:uri "/baz/something"}) (let [summary (request-summary http-metrics) foo-summary (-> summary :routes (get "foo-:foo")) bar-summary (-> summary :routes (get "bar-:bar"))] (is (= [:total "bar-:bar" "foo-:foo" :other] (mapv :route-id (:sorted-routes summary)))) (is (= 4 (-> summary :routes :total :count))) (is (= 2 (:count foo-summary))) (is (= 1 (:count bar-summary))) (is (= 1 (-> summary :routes :other :count))) (is (<= 20 (:mean bar-summary))) (is (<= 20 (:aggregate bar-summary))) (is (<= 5 (:mean foo-summary))) (is (<= 10 (:aggregate foo-summary))) (is (> (:mean bar-summary) (:mean foo-summary))) (is (> (:mean bar-summary) (-> summary :routes :total :mean)))))))