From 007f8c00b27a4e144611c2a8284dd5dec196157a Mon Sep 17 00:00:00 2001 From: Yann Esposito Date: Tue, 17 Feb 2015 11:40:29 +0100 Subject: [PATCH] initial commit --- .gitignore | 11 + LICENSE | 21 + README.md | 164 ++ docs/uberdoc.html | 3219 ++++++++++++++++++++++++++++++++ project.clj | 12 + src/bigbrother/core.clj | 126 ++ src/bigbrother/counter.clj | 39 + src/bigbrother/max_metrics.clj | 35 + src/bigbrother/metrics.clj | 37 + src/bigbrother/timer.clj | 86 + test/bigbrother/core_test.clj | 120 ++ 11 files changed, 3870 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docs/uberdoc.html create mode 100644 project.clj create mode 100644 src/bigbrother/core.clj create mode 100644 src/bigbrother/counter.clj create mode 100644 src/bigbrother/max_metrics.clj create mode 100644 src/bigbrother/metrics.clj create mode 100644 src/bigbrother/timer.clj create mode 100644 test/bigbrother/core_test.clj diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c53038e --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +/target +/classes +/checkouts +pom.xml +pom.xml.asc +*.jar +*.class +/.lein-* +/.nrepl-port +.hgignore +.hg/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d216c3c --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) [year] [fullname] + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..60cab1f --- /dev/null +++ b/README.md @@ -0,0 +1,164 @@ +# Big Brother + +A Clojure library designed to monitor things and retrieve some metrics. + +It could send the metrics to [Riemann](http://riemann.io). + +Add this dependency to your `project.clj`: + +~~~ +[bigbrother "0.1.0"] +~~~ + +## Usage + +Full example: + +~~~ {.clojure} +(ns myproject.core + (:require [bigbrother.core :as bb])) + +... +(def riemann-host "localhost") +(def riemann-service "bb") +(def nb-ms-metrics 1000) ; send metrics info every second +(bb/init-metrics + {:read-book (bb/warn-level 10 100) ; <- less than 10ms is ok + :write-notes (bb/warn-level 10 100) ; <- warn between 10ms and 100ms + :feel-free (bb/warn-level 10 100) ; <- critical if more than 100ms + :free-time (bb/warn-level 10 100) ; <- normal if less than 10ms + :work-as-usual (bb/rev-warn-level 100 10) ; <- normal if more than 100 critical <10 + :total (bb/warn-level 100 500) ; <- 0..100 normal, 0..500 warn, >500 critical + :nb (bb/rev-warn-level 5 1) <- less than 1 critical, less than 5 warn + } + nb-ms-metrics + riemann-host + riemann-service) + +;; If you log metrics not declared in init-metrics, they will show as always "ok" + +... + +(defn function-called-frequently + [] + (bb/big-brother-is-watching-you) ; <-- you must start the timer + (when (has-book?) + (read-book) ; <-- do your action + (bb/log-time :read-book) ; <-- tell the timer you've done something + (bb/log-counter :book) ; <-- increment the counter :book + ) + (write-notes) + (bb/log-time :write-notes) + (let [time-felt-free (feel-free)] + (bb/log-time :feel-free) + (bb/log-mmetric :freetime time-felt-free)) + (work-as-usual) + (bb/log-time :work-as-usual) + (bb/telescreen-off)) ; <-- end the timer loop to flush times. +~~~ + +### Init + +The `init-metrics` function takes: + +- a map containing metrics name for keys and warning level functions as values +- the nb of ms to wait before sending new metrics +- a riemann host (if `nil` won't use riemann) +- a riemann service name. + +That will send data to riemann every nb-ms-metrics ms. + +~~~ +(bb/init-metrics + {:read-book (bb/warn-level 10 100) ; <- less than 10ms is ok + :write-notes (bb/warn-level 10 100) ; <- warn between 10ms and 100ms + :feel-free (bb/warn-level 10 100) ; <- critical if more than 100ms + :free-time (bb/warn-level 10 100) ; <- normal if less than 10ms + :work-as-usual (bb/rev-warn-level 100 10) ; <- normal if more than 100 critical <10 + :total (bb/warn-level 100 500) ; <- 0..100 normal, 0..500 warn, >500 critical + :nb (bb/rev-warn-level 5 1) <- less than 1 critical, less than 5 warn + } + nb-ms-metrics + riemann-host + riemann-service) +~~~ + +### Timers + +~~~ +(do + (action) + (bb/log-time :action)) +~~~ + +Will tell big brother that the action :action finished. +Generally you use timers like this: + +~~~ +(do + (bb/big-brother-is-watching-you) + (action1) + (bb/log-time :action1) + (action1) + (bb/log-time :action1) + (bb/telescreen-off)) +~~~ + +You declare a start time which will start the chrono. +Then each time you decalare a `log-time` it will add the time taken to the action name. + +Once you finished your course of action you declare the timer-loop as finished. +And you stop the chrono up until the next `(bb/big-brother-is-watching-you)`. + +In the end, the value will be the mean of the time taken by each action during the +big loop. + +### Counters + +Counter can be called: + +~~~ +(do + (bb/log-counter :foo) + (bb/log-counter :bar 3)) +~~~ + +Will increment :foo by 1 and :bar by 3. +In the end, the result will be the mean of the counter number by second. + +### Metrics + +Metrics are quite similar to counters. +The main difference is that there is no notion of increment. +You must always provide a value for a metrics. +The resume will contains its mean normalized by second. + +~~~ +(do + (bb/log-metric :foo 0.5) + (bb/log-metric :foo 2.5) + (bb/log-metric :foo 3)) +~~~ + +In the resume you'll get `{:foo 2.0}` (`(0.5 + 2.5 + 3)/3`). + + +### Max Metrics + +Sometime you might want to get the maximum of some values during an interval of time. + +~~~ +(do + (bb/log-mmetric :foo 5) + (bb/log-mmetric :foo 1) + (bb/log-mmetric :foo 3)) +~~~ + +That will display `{:foo 5}` in the resume. + +## License + +Copyright © 2015 Yann Esposito + +Distributed under the Eclipse Public License either version 1.0 or (at +your option) any later version. diff --git a/docs/uberdoc.html b/docs/uberdoc.html new file mode 100644 index 0000000..95e68fd --- /dev/null +++ b/docs/uberdoc.html @@ -0,0 +1,3219 @@ + +bigbrother -- Marginalia

bigbrother

0.1.0-SNAPSHOT


Periodically send metrics

+

dependencies

org.clojure/clojure
1.6.0
org.clojure/data.json
0.2.5
org.clojure/tools.logging
0.3.1
org.clojure/algo.generic
0.1.2
org.clojure/test.check
0.7.0
riemann-clojure-client
0.3.2
overtone/at-at
1.2.0



(this space intentionally left almost blank)

namespaces

 

bigbrother.core

toc

This file provide helpers to manage time spend in functions

+
(ns bigbrother.core
+  (:require [clojure.tools.logging :as log]
+            [clojure.data.json :as json]
+            [overtone.at-at :refer [every mk-pool]]
+            [riemann.client :as r]
+            [clojure.algo.generic.functor :refer [fmap]]
+            [bigbrother.timer :as timer]
+            [bigbrother.counter :as counter]
+            [bigbrother.metrics :as metrics]
+            [bigbrother.max-metrics :as max-metrics]))

Atoms

+
(def pool (atom nil)) ;; pool for async

pool for async

+
(def default-map (atom #{}))
+(def riemann-conn (atom nil))
+(def riemann-service (atom "supercell"))
+(def level-by-key (atom nil))
+
(def n (atom 0))      ;; a number

a number

+
+
(def log-time timer/log-time)
+(def log-counter counter/log-counter)
+(def log-metric metrics/log-metric)
+(def log-mmetric max-metrics/log-mmetric)
+
(defn timer-loop-finished []
+  ;; increment the number of loop
+  (swap! n inc)
+  (timer/finish-timer-loop)
+  (max-metrics/loop-finished)
+  (metrics/loop-finished)
+  (counter/loop-finished))

Starting the timer

+ +

---- aliases

+ +

Starting the timer

+
(defn big-brother-is-watching-you  []
+  (timer/log-time :start))
+(def telescreen-on  big-brother-is-watching-you)

End the timer chrono

+ +

End the timer chrono

+
(def welcome-in-miniluv  timer-loop-finished)
+(def telescreen-off  timer-loop-finished)

+Riemann Warn Level

+

warn level between two numbers

+
(defn warn-level
+  [warn crit]
+  (fn [v] (cond (neg? v) "critical"
+                (>= v crit) "critical"
+                (>= v warn) "warning"
+                :else "ok")))
+(defn rev-warn-level
+  [warn crit]
+  (fn [v] (cond (<= v crit) "critical"
+                (<= v warn) "warning"
+                :else "ok")))
+
(defn ok [] (fn [_] "ok"))
+(defn warning [] (fn [_] "warning"))
+(defn critical [] (fn [_] "critical"))
+
(defn- to-riemann-event [[k v]]
+  (when (number? v)
+    (let [lvl-fn (get @level-by-key k)
+          level  (if lvl-fn (lvl-fn v) "ok")]
+      {:service (str @riemann-service " " (name k) (subs (str k) 1))
+       :state level 
+       :metric v})))
+
(defn send-to-riemann [m]
+  (let [result-map (into @default-map m)
+        metric-data {:service @riemann-service
+                     :state "ok"}
+        events (remove nil? (map to-riemann-event result-map))
+        ]
+    (when @riemann-conn
+      (r/send-events @riemann-conn events))))
+
(defn reset-accumulators! []
+  (counter/reset-acc!)
+  (metrics/reset-acc!)
+  (max-metrics/reset-acc!)
+  (timer/reset-acc!)
+  (reset! n 0))
+
(defn reset-all-atoms! []
+  (reset! timer/times [])
+  (reset! metrics/metrics {})
+  (reset! max-metrics/mmetrics {})
+  (reset! counter/counters {})
+  (reset-accumulators!))
+
(defn resume-map [nb-ms]
+  (let [n-by-sec  (float (/ @n (/ nb-ms 1000)))
+        basic     {:nb n-by-sec}]
+    (reduce into basic [(metrics/resume @n)
+                        (max-metrics/resume)
+                        (timer/resume)
+                        (counter/resume nb-ms)])))

display the time at most every 10s

+
(defn display-time
+  [nb-ms]
+  (let [result (resume-map nb-ms)]
+    (log/info (json/write-str result))
+    (send-to-riemann result)
+    (reset-accumulators!)))

init-metrics

+ +
init-map :: Map Keyword (v -> ERROR_LEVEL)
+
+
(defn init-metrics
+  [init-map nb-ms-metrics riemann-host riemann-service-name]
+  (reset! default-map (reduce into {}
+                              (map (fn [k] {k -1} )
+                                   (conj (keys init-map) :total))))
+  (reset! level-by-key init-map)
+  (reset! pool (mk-pool))
+  (reset! riemann-service riemann-service-name)
+  (when riemann-host
+    (reset! riemann-conn (r/tcp-client {:host riemann-host})))
+  (every nb-ms-metrics (fn [] (display-time nb-ms-metrics)) @pool))
 

bigbrother.counter

toc
+
(ns bigbrother.counter
+  (:require
+   [clojure.algo.generic.functor :refer [fmap]]))

+COUNTERS

+

Monoid instance of `sumcounter`

+ +

a sum time is a list of couple {name nb}

+
(def empty-sumcounter {})
+(defn add-sumcounter [st st2]
+  (merge-with + st st2))

Counters atoms

+
(def counters (atom {}))
+(def sumcounters (atom empty-sumcounter))

Counters (the mean is given by default)

+ +

declare a specific counters (not time)

+ +

declare a specific counters (not time) and returns the first parameter

+
(defn- set-counter! [k v] (swap! counters add-sumcounter {k v}))
+(defn log-counter
+  ([k] (set-counter! k 1))
+  ([k v] (set-counter! k v)))
+(defn log-counter->
+  [x k v]
+  (log-counter k v)
+  x)
+
(defn loop-finished []
+  ;; aggreate sumcounter
+  (swap! sumcounters add-sumcounter @counters)
+  (reset! counters {}))
+
(defn reset-acc! []
+  (reset! sumcounters empty-sumcounter))
+
(defn resume [nb-ms]
+  (fmap #(float (/ % (/ nb-ms 1000))) @sumcounters))
 

bigbrother.max-metrics

toc
+
(ns bigbrother.max-metrics)

+Max Metrics

+

Monoid instance of `maxmetrics`

+ +

a sum time is a list of couple {name metric-value}

+
(def empty-maxmetric {})
+(defn add-maxmetrics [st st2]
+  (merge-with max st st2))
+
(def mmetrics (atom {})) ;; timestamps

timestamps

+
(def maxmetrics (atom empty-maxmetric)) ;; time spent by type

time spent by type

+

Max Metrics (metrics to do a max between them instead of a mean)

+ +

declare a specific max metrics (not time)

+ +

declare a specific max metrics (not time) and returns the first parameter

+
(defn- set-mmetric! [k v] (swap! mmetrics add-maxmetrics {k v}))
+(defn log-mmetric
+  [k v]
+  (set-mmetric! k v))
+(defn log-mmetric->
+  [x k v]
+  (log-mmetric k v)
+  x)
+
(defn loop-finished []
+  ;; aggreate maxmetrics
+  (swap! maxmetrics add-maxmetrics @mmetrics)
+  (reset! mmetrics {}))
+
(defn reset-acc! []
+  (reset! maxmetrics empty-maxmetric))
+
(defn resume [] @maxmetrics)
 

bigbrother.metrics

toc
+
(ns bigbrother.metrics
+  (:require [clojure.algo.generic.functor :refer [fmap]]))

+Sum Metrics

+

Monoid instance of `summetrics`

+ +

a sum time is a list of couple {name metric-value}

+
(def empty-summetric {})
+(defn add-summetrics [st st2]
+  (merge-with + st st2))
+
(def metrics (atom {})) ;; timestamps

timestamps

+
(def summetrics (atom empty-summetric)) ;; time spent by type

time spent by type

+

Metrics (the mean is given by default)

+ +

declare a specific metrics (not time)

+ +

declare a specific metrics (not time) and returns the first parameter

+
(defn- set-metric! [k v] (swap! metrics add-summetrics {k v}))
+(defn log-metric
+  [k v]
+  (set-metric! k v))
+(defn log-metric->
+  [x k v]
+  (log-metric k v)
+  x)
+
(defn loop-finished []
+  ;; aggreate summetrics
+  (swap! summetrics add-summetrics @metrics)
+  (reset! metrics {}))
+
(defn reset-acc! []
+  (reset! summetrics empty-summetric))
+
(defn resume [n]
+  (fmap #(float (/ % n)) @summetrics))
 

bigbrother.timer

toc
+
(ns bigbrother.timer)

+TIMERS

+

Monoid instance of `sumtimes`

+ +

a sum time is a list of couple [name [timespent nb]]

+
(defn ts-name [x] (first x))
+(defn ts-timespent [x] (first (second x)))
+(defn ts-nb [x] (second (second x)))
+
(def empty-sumtime [])
+(defn- add-one-sumtime [st st2]
+  [(ts-name st) [(+ (ts-timespent st) (ts-timespent st2))
+                 (+ (ts-nb st) (ts-nb st2))]])
+(defn add-sumtimes [st st2]
+  (cond (empty? st) st2
+        (empty? st2) st
+        :else (map add-one-sumtime st st2)))
+(defn fmap-sumtimes [f st]
+  (map (fn [v] [(first v) [(f (ts-timespent v))
+                           (ts-nb v)]]) st))

Given a sumtimes returns a map {key timespent}'

+
(defn normalized-map-from-sumtimes
+  [st]
+  (reduce #(merge-with + %1 %2) {}
+          (map (fn [v] {(ts-name v)
+                        (/ (ts-timespent v)
+                           (ts-nb v))}) st)))

timers atoms

+
(def times (atom [])) ;; timestamps

timestamps

+
(def sumtimes (atom empty-sumtime)) ;; time spent by type

time spent by type

+

Timer

+ +

declare the action named k finished

+ +

declare the action named k finished and returned object x

+
(defn- set-value! [k v] (swap! times conj [k v]))
+(defn log-time
+  [k]
+  (set-value! k (System/nanoTime)))
+(defn log-time->
+  [x k]
+  (log-time k)
+  x)

get time spent during one step

+
(defn- show-one-step
+  [x]
+  (if (< (count x) 2)
+    {:nothing 0}
+    (let [from (second (first x))
+          k    (first  (second x))
+          to   (second (second x))]
+      [k [(- to from) 1]])))

from a list of timestamp generate a sumtime

+
(defn timespent
+  [times-array]
+  (map show-one-step (partition 2 1 times-array)))

from a list of timestamp generate the total time spent

+
(defn total-time
+  [times-array]
+  (- (second (last times-array))
+     (second (first times-array))))

convert from nanoseconds to milliseconds

+
(defn to-milliseconds
+  [times-array]
+  (fmap-sumtimes #(float (/ % 1000000)) times-array))
+
(defn finish-timer-loop []
+  ;; Convert actual timestamps to sumtimes and aggregate them
+  (if (> (count @times) 1)
+    (let [difftime (timespent @times)
+          total    (total-time @times)
+          res      (to-milliseconds (conj difftime [:total [total 1]]))]
+      (swap! sumtimes add-sumtimes res)))
+  (reset! times []))
+
(defn reset-acc! []
+  (reset! sumtimes empty-sumtime))
+
(defn resume []
+  (normalized-map-from-sumtimes @sumtimes))
 
Generated by Marginalia.  Syntax highlighting provided by Alex Gorbatchev's SyntaxHighlighter
  • bigbrother.core
  • bigbrother.counter
  • bigbrother.max-metrics
  • bigbrother.metrics
  • bigbrother.timer
\ No newline at end of file diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..e60c137 --- /dev/null +++ b/project.clj @@ -0,0 +1,12 @@ +(defproject bigbrother "0.1.0-SNAPSHOT" + :description "Periodically send metrics" + :url "http://github.com/yogsototh/bigbrother" + :license {:name "MIT" + :url "http://chosealicense.com/licenses/mit"} + :dependencies [[org.clojure/clojure "1.6.0"] + [org.clojure/data.json "0.2.5"] + [org.clojure/tools.logging "0.3.1"] + [org.clojure/algo.generic "0.1.2"] + [org.clojure/test.check "0.7.0"] + [riemann-clojure-client "0.3.2"] + [overtone/at-at "1.2.0"]]) diff --git a/src/bigbrother/core.clj b/src/bigbrother/core.clj new file mode 100644 index 0000000..f7780ca --- /dev/null +++ b/src/bigbrother/core.clj @@ -0,0 +1,126 @@ +(ns bigbrother.core + "This file provide helpers to manage time spend in functions" + (:require [clojure.tools.logging :as log] + [clojure.data.json :as json] + [overtone.at-at :refer [every mk-pool]] + [riemann.client :as r] + [clojure.algo.generic.functor :refer [fmap]] + + [bigbrother.timer :as timer] + [bigbrother.counter :as counter] + [bigbrother.metrics :as metrics] + [bigbrother.max-metrics :as max-metrics] + )) + +;; Atoms +(def pool (atom nil)) ;; pool for async +(def default-map (atom #{})) +(def riemann-conn (atom nil)) +(def riemann-service (atom "supercell")) +(def level-by-key (atom nil)) + +(def n (atom 0)) ;; a number + +(def log-time timer/log-time) +(def log-counter counter/log-counter) +(def log-metric metrics/log-metric) +(def log-mmetric max-metrics/log-mmetric) + +(defn timer-loop-finished [] + ;; increment the number of loop + (swap! n inc) + (timer/finish-timer-loop) + (max-metrics/loop-finished) + (metrics/loop-finished) + (counter/loop-finished)) + +;; ---- aliases +(defn big-brother-is-watching-you "Starting the timer" [] + (timer/log-time :start)) +(def telescreen-on "Starting the timer" big-brother-is-watching-you) + +(def welcome-in-miniluv "End the timer chrono" timer-loop-finished) +(def telescreen-off "End the timer chrono" timer-loop-finished) + +;; ------------------------------------------------------------------------------ +;; Riemann Warn Level + +(defn warn-level + "warn level between two numbers" + [warn crit] + (fn [v] (cond (neg? v) "critical" + (>= v crit) "critical" + (>= v warn) "warning" + :else "ok"))) +(defn rev-warn-level + [warn crit] + (fn [v] (cond (<= v crit) "critical" + (<= v warn) "warning" + :else "ok"))) + +(defn ok [] (fn [_] "ok")) +(defn warning [] (fn [_] "warning")) +(defn critical [] (fn [_] "critical")) + +(defn- to-riemann-event [[k v]] + (when (number? v) + (let [lvl-fn (get @level-by-key k) + level (if lvl-fn (lvl-fn v) "ok")] + {:service (str @riemann-service " " (name k) (subs (str k) 1)) + :state level + :metric v}))) + +(defn send-to-riemann [m] + (let [result-map (into @default-map m) + metric-data {:service @riemann-service + :state "ok"} + events (remove nil? (map to-riemann-event result-map)) + ] + (when @riemann-conn + (r/send-events @riemann-conn events)))) + + +(defn reset-accumulators! [] + (counter/reset-acc!) + (metrics/reset-acc!) + (max-metrics/reset-acc!) + (timer/reset-acc!) + (reset! n 0)) + +(defn reset-all-atoms! [] + (reset! timer/times []) + (reset! metrics/metrics {}) + (reset! max-metrics/mmetrics {}) + (reset! counter/counters {}) + (reset-accumulators!)) + +(defn resume-map [nb-ms] + (let [n-by-sec (float (/ @n (/ nb-ms 1000))) + basic {:nb n-by-sec}] + (reduce into basic [(metrics/resume @n) + (max-metrics/resume) + (timer/resume) + (counter/resume nb-ms)]))) + +(defn display-time + "display the time at most every 10s" + [nb-ms] + (let [result (resume-map nb-ms)] + (log/info (json/write-str result)) + (send-to-riemann result) + (reset-accumulators!))) + +(defn init-metrics + "## init-metrics + + init-map :: Map Keyword (v -> ERROR_LEVEL)" + [init-map nb-ms-metrics riemann-host riemann-service-name] + (reset! default-map (reduce into {} + (map (fn [k] {k -1} ) + (conj (keys init-map) :total)))) + (reset! level-by-key init-map) + (reset! pool (mk-pool)) + (reset! riemann-service riemann-service-name) + (when riemann-host + (reset! riemann-conn (r/tcp-client {:host riemann-host}))) + (every nb-ms-metrics (fn [] (display-time nb-ms-metrics)) @pool)) diff --git a/src/bigbrother/counter.clj b/src/bigbrother/counter.clj new file mode 100644 index 0000000..04c632b --- /dev/null +++ b/src/bigbrother/counter.clj @@ -0,0 +1,39 @@ +(ns bigbrother.counter + (:require + [clojure.algo.generic.functor :refer [fmap]])) + +;; ------------------------------------------------------------------------------ +;; COUNTERS + +;; ## Monoid instance of `sumcounter` +;; a sum time is a list of couple `{name nb}` +(def empty-sumcounter {}) +(defn add-sumcounter [st st2] + (merge-with + st st2)) + +;; Counters atoms +(def counters (atom {})) +(def sumcounters (atom empty-sumcounter)) + +;; Counters (the mean is given by default) +(defn- set-counter! [k v] (swap! counters add-sumcounter {k v})) +(defn log-counter + "declare a specific counters (not time)" + ([k] (set-counter! k 1)) + ([k v] (set-counter! k v))) +(defn log-counter-> + "declare a specific counters (not time) and returns the first parameter" + [x k v] + (log-counter k v) + x) + +(defn loop-finished [] + ;; aggreate sumcounter + (swap! sumcounters add-sumcounter @counters) + (reset! counters {})) + +(defn reset-acc! [] + (reset! sumcounters empty-sumcounter)) + +(defn resume [nb-ms] + (fmap #(float (/ % (/ nb-ms 1000))) @sumcounters)) diff --git a/src/bigbrother/max_metrics.clj b/src/bigbrother/max_metrics.clj new file mode 100644 index 0000000..95bac64 --- /dev/null +++ b/src/bigbrother/max_metrics.clj @@ -0,0 +1,35 @@ +(ns bigbrother.max-metrics) + +;; ------------------------------------------------------------------------------ +;; Max Metrics + +;; ## Monoid instance of `maxmetrics` +;; a sum time is a list of couple `{name metric-value}` +(def empty-maxmetric {}) +(defn add-maxmetrics [st st2] + (merge-with max st st2)) + +(def mmetrics (atom {})) ;; timestamps +(def maxmetrics (atom empty-maxmetric)) ;; time spent by type + +;; Max Metrics (metrics to do a max between them instead of a mean) +(defn- set-mmetric! [k v] (swap! mmetrics add-maxmetrics {k v})) +(defn log-mmetric + "declare a specific max metrics (not time)" + [k v] + (set-mmetric! k v)) +(defn log-mmetric-> + "declare a specific max metrics (not time) and returns the first parameter" + [x k v] + (log-mmetric k v) + x) + +(defn loop-finished [] + ;; aggreate maxmetrics + (swap! maxmetrics add-maxmetrics @mmetrics) + (reset! mmetrics {})) + +(defn reset-acc! [] + (reset! maxmetrics empty-maxmetric)) + +(defn resume [] @maxmetrics) diff --git a/src/bigbrother/metrics.clj b/src/bigbrother/metrics.clj new file mode 100644 index 0000000..fa02ab0 --- /dev/null +++ b/src/bigbrother/metrics.clj @@ -0,0 +1,37 @@ +(ns bigbrother.metrics + (:require [clojure.algo.generic.functor :refer [fmap]])) + +;; ------------------------------------------------------------------------------ +;; Sum Metrics + +;; ## Monoid instance of `summetrics` +;; a sum time is a list of couple `{name metric-value}` +(def empty-summetric {}) +(defn add-summetrics [st st2] + (merge-with + st st2)) + +(def metrics (atom {})) ;; timestamps +(def summetrics (atom empty-summetric)) ;; time spent by type + +;; Metrics (the mean is given by default) +(defn- set-metric! [k v] (swap! metrics add-summetrics {k v})) +(defn log-metric + "declare a specific metrics (not time)" + [k v] + (set-metric! k v)) +(defn log-metric-> + "declare a specific metrics (not time) and returns the first parameter" + [x k v] + (log-metric k v) + x) + +(defn loop-finished [] + ;; aggreate summetrics + (swap! summetrics add-summetrics @metrics) + (reset! metrics {})) + +(defn reset-acc! [] + (reset! summetrics empty-summetric)) + +(defn resume [n] + (fmap #(float (/ % n)) @summetrics)) diff --git a/src/bigbrother/timer.clj b/src/bigbrother/timer.clj new file mode 100644 index 0000000..dbdcf43 --- /dev/null +++ b/src/bigbrother/timer.clj @@ -0,0 +1,86 @@ +(ns bigbrother.timer) + +;; ------------------------------------------------------------------------------ +;; TIMERS + +;; ## Monoid instance of `sumtimes` +;; a sum time is a list of couple `[name [timespent nb]]` +(defn ts-name [x] (first x)) +(defn ts-timespent [x] (first (second x))) +(defn ts-nb [x] (second (second x))) + +(def empty-sumtime []) +(defn- add-one-sumtime [st st2] + [(ts-name st) [(+ (ts-timespent st) (ts-timespent st2)) + (+ (ts-nb st) (ts-nb st2))]]) +(defn add-sumtimes [st st2] + (cond (empty? st) st2 + (empty? st2) st + :else (map add-one-sumtime st st2))) +(defn fmap-sumtimes [f st] + (map (fn [v] [(first v) [(f (ts-timespent v)) + (ts-nb v)]]) st)) + +(defn normalized-map-from-sumtimes + "Given a `sumtimes` returns a map `{key timespent}`'" + [st] + (reduce #(merge-with + %1 %2) {} + (map (fn [v] {(ts-name v) + (/ (ts-timespent v) + (ts-nb v))}) st))) +;; timers atoms +(def times (atom [])) ;; timestamps +(def sumtimes (atom empty-sumtime)) ;; time spent by type + +;; Timer +(defn- set-value! [k v] (swap! times conj [k v])) +(defn log-time + "declare the action named `k` finished" + [k] + (set-value! k (System/nanoTime))) +(defn log-time-> + "declare the action named `k` finished and returned object `x`" + [x k] + (log-time k) + x) + +(defn- show-one-step + "get time spent during one step" + [x] + (if (< (count x) 2) + {:nothing 0} + (let [from (second (first x)) + k (first (second x)) + to (second (second x))] + [k [(- to from) 1]]))) + +(defn timespent + "from a list of timestamp generate a sumtime" + [times-array] + (map show-one-step (partition 2 1 times-array))) + +(defn total-time + "from a list of timestamp generate the total time spent" + [times-array] + (- (second (last times-array)) + (second (first times-array)))) + +(defn to-milliseconds + "convert from nanoseconds to milliseconds" + [times-array] + (fmap-sumtimes #(float (/ % 1000000)) times-array)) + +(defn finish-timer-loop [] + ;; Convert actual timestamps to sumtimes and aggregate them + (if (> (count @times) 1) + (let [difftime (timespent @times) + total (total-time @times) + res (to-milliseconds (conj difftime [:total [total 1]]))] + (swap! sumtimes add-sumtimes res))) + (reset! times [])) + +(defn reset-acc! [] + (reset! sumtimes empty-sumtime)) + +(defn resume [] + (normalized-map-from-sumtimes @sumtimes)) diff --git a/test/bigbrother/core_test.clj b/test/bigbrother/core_test.clj new file mode 100644 index 0000000..22d7fc4 --- /dev/null +++ b/test/bigbrother/core_test.clj @@ -0,0 +1,120 @@ +(ns bigbrother.core-test + (:require + [bigbrother.core :refer :all] + + [clojure.test :refer :all] + [clojure.test.check.generators :as gen] + [clojure.test.check.properties :as prop] + [clojure.test.check.clojure-test :refer [defspec]])) + +(deftest timer-test + (do + (reset-all-atoms!) + ;; first loop + (telescreen-on) + (Thread/sleep 30) + (log-time :x1) + (Thread/sleep 30) + (log-time :x2) + (Thread/sleep 30) + (log-time :end) + (telescreen-off) + ;; second loop + (log-time :start) + (Thread/sleep 30) + (log-time :x2) + (Thread/sleep 30) + (log-time :end) + (telescreen-off) + (let [result (resume-map 1000)] + (is (contains? result :nb)) + (is (contains? result :x1)) + (is (contains? result :x2)) + (is (>= (:total result) (:x2 result)))))) + +(deftest check-metrics + (do + (reset-all-atoms!) + ;; first loop + (log-metric :foo 1) + (log-metric :bar 3) + (timer-loop-finished) + (log-metric :foo 2) + (log-metric :bar 2) + (timer-loop-finished) + (log-metric :foo 3) + (log-metric :bar 1) + (timer-loop-finished) + (let [result (resume-map 1000)] + (is (contains? result :foo)) + (is (contains? result :bar)) + (is (contains? result :nb)) + (is (= 2.0 (:foo result))) + (is (= 2.0 (:bar result))) + (is (= 3.0 (:nb result)))))) + +(deftest check-mmetrics + (do + (reset-all-atoms!) + (log-mmetric :foo 80) + (log-mmetric :bar 4) + (timer-loop-finished) + (log-mmetric :foo 50) + (log-mmetric :bar 5) + (timer-loop-finished) + (log-mmetric :foo 60) + (log-mmetric :bar 6) + (timer-loop-finished) + (let [result (resume-map 1000)] + (is (contains? result :foo)) + (is (contains? result :bar)) + (is (contains? result :nb)) + (is (= 80 (:foo result))) + (is (= 6 (:bar result))) + (is (= 3.0 (:nb result)))))) + +(deftest check-counters + (do + (reset-all-atoms!) + ;; first loop + (log-counter :foo) + (log-counter :foo) + (log-counter :bar 3) + (timer-loop-finished) + (log-counter :foo) + (log-counter :bar 2) + (timer-loop-finished) + (log-counter :foo) + (log-counter :bar 1) + (timer-loop-finished) + (let [result (resume-map 1000)] + (is (contains? result :foo)) + (is (= 4.0 (:foo result))) + (is (= 6.0 (:bar result))) + (is (= 3.0 (:nb result)))))) + +(defspec check-all-keys-exists + 30 + (prop/for-all + [actions (gen/such-that #(> (count %) 1) + (gen/vector gen/keyword))] + (do + (reset-all-atoms!) + (doall (map (fn [a] (do (log-time a) (Thread/sleep 10))) + actions)) + (timer-loop-finished)) + (let [result (resume-map 1000)] + (every? #(contains? result %) (rest actions))))) + +(defspec check-total-is-greater-than-each-input + 30 + (prop/for-all + [actions (gen/such-that #(> (count %) 1) + (gen/vector gen/keyword))] + (do + (reset-all-atoms!) + (doall (map (fn [a] (do (log-time a) (Thread/sleep 10))) + actions)) + (timer-loop-finished)) + (let [result (resume-map 1000)] + (every? #(>= (:total result) (get result %)) (rest actions)))))