initial commit

.gitignore

@@ -0,0 +1,11 @@
+/target
+/classes
+/checkouts
+pom.xml
+pom.xml.asc
+*.jar
+*.class
+/.lein-*
+/.nrepl-port
+.hgignore
+.hg/

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,164 @@
+# Big Brother
+
+A Clojure library designed to monitor things and retrieve some metrics.
+
+It could send the metrics to [Riemann](http://riemann.io).
+
+Add this dependency to your `project.clj`:
+
+~~~
+[bigbrother "0.1.0"]
+~~~
+
+## Usage
+
+Full example:
+
+~~~ {.clojure}
+(ns myproject.core
+    (:require [bigbrother.core :as bb]))
+
+...
+(def riemann-host "localhost")
+(def riemann-service "bb")
+(def nb-ms-metrics 1000) ; send metrics info every second
+(bb/init-metrics
+    {:read-book     (bb/warn-level 10 100) ; <- less than 10ms is ok
+     :write-notes   (bb/warn-level 10 100) ; <- warn between 10ms and 100ms
+     :feel-free     (bb/warn-level 10 100) ; <- critical if more than 100ms
+     :free-time     (bb/warn-level 10 100) ; <- normal if less than 10ms
+     :work-as-usual (bb/rev-warn-level 100 10) ; <- normal if more than 100 critical <10
+     :total         (bb/warn-level 100 500) ; <- 0..100 normal, 0..500 warn, >500 critical
+     :nb            (bb/rev-warn-level 5 1) <- less than 1 critical, less than 5 warn
+     }
+    nb-ms-metrics
+    riemann-host
+    riemann-service)
+
+;; If you log metrics not declared in init-metrics, they will show as always "ok"
+
+...
+
+(defn function-called-frequently
+    []
+    (bb/big-brother-is-watching-you) ; <-- you must start the timer
+    (when (has-book?)
+        (read-book) ; <-- do your action
+        (bb/log-time :read-book) ; <-- tell the timer you've done something
+        (bb/log-counter :book) ; <-- increment the counter :book
+        )
+    (write-notes)
+    (bb/log-time :write-notes)
+    (let [time-felt-free (feel-free)]
+        (bb/log-time :feel-free)
+        (bb/log-mmetric :freetime time-felt-free))
+    (work-as-usual)
+    (bb/log-time :work-as-usual)
+    (bb/telescreen-off)) ; <-- end the timer loop to flush times.
+~~~
+
+### Init
+
+The `init-metrics` function takes:
+
+- a map containing metrics name for keys and warning level functions as values
+- the nb of ms to wait before sending new metrics
+- a riemann host (if `nil` won't use riemann)
+- a riemann service name.
+
+That will send data to riemann every nb-ms-metrics ms.
+
+~~~
+(bb/init-metrics
+    {:read-book     (bb/warn-level 10 100) ; <- less than 10ms is ok
+     :write-notes   (bb/warn-level 10 100) ; <- warn between 10ms and 100ms
+     :feel-free     (bb/warn-level 10 100) ; <- critical if more than 100ms
+     :free-time     (bb/warn-level 10 100) ; <- normal if less than 10ms
+     :work-as-usual (bb/rev-warn-level 100 10) ; <- normal if more than 100 critical <10
+     :total         (bb/warn-level 100 500) ; <- 0..100 normal, 0..500 warn, >500 critical
+     :nb            (bb/rev-warn-level 5 1) <- less than 1 critical, less than 5 warn
+     }
+    nb-ms-metrics
+    riemann-host
+    riemann-service)
+~~~
+
+### Timers
+
+~~~
+(do
+    (action)
+    (bb/log-time :action))
+~~~
+
+Will tell big brother that the action :action finished.
+Generally you use timers like this:
+
+~~~
+(do
+    (bb/big-brother-is-watching-you)
+    (action1)
+    (bb/log-time :action1)
+    (action1)
+    (bb/log-time :action1)
+    (bb/telescreen-off))
+~~~
+
+You declare a start time which will start the chrono.
+Then each time you decalare a `log-time` it will add the time taken to the action name.
+
+Once you finished your course of action you declare the timer-loop as finished.
+And you stop the chrono up until the next `(bb/big-brother-is-watching-you)`.
+
+In the end, the value will be the mean of the time taken by each action during the
+big loop.
+
+### Counters
+
+Counter can be called:
+
+~~~
+(do
+  (bb/log-counter :foo)
+  (bb/log-counter :bar 3))
+~~~
+
+Will increment :foo by 1 and :bar by 3.
+In the end, the result will be the mean of the counter number by second.
+
+### Metrics
+
+Metrics are quite similar to counters.
+The main difference is that there is no notion of increment.
+You must always provide a value for a metrics.
+The resume will contains its mean normalized by second.
+
+~~~
+(do
+    (bb/log-metric :foo 0.5)
+    (bb/log-metric :foo 2.5)
+    (bb/log-metric :foo 3))
+~~~
+
+In the resume you'll get `{:foo 2.0}` (`(0.5 + 2.5 + 3)/3`).
+
+
+### Max Metrics
+
+Sometime you might want to get the maximum of some values during an interval of time.
+
+~~~
+(do
+  (bb/log-mmetric :foo 5)
+  (bb/log-mmetric :foo 1)
+  (bb/log-mmetric :foo 3))
+~~~
+
+That will display `{:foo 5}` in the resume.
+
+## License
+
+Copyright © 2015 Yann Esposito
+
+Distributed under the Eclipse Public License either version 1.0 or (at
+your option) any later version.

project.clj

@@ -0,0 +1,12 @@
+(defproject bigbrother "0.1.0-SNAPSHOT"
+  :description "Periodically send metrics"
+  :url "http://github.com/yogsototh/bigbrother"
+  :license {:name "MIT"
+            :url "http://chosealicense.com/licenses/mit"}
+  :dependencies [[org.clojure/clojure "1.6.0"]
+                 [org.clojure/data.json "0.2.5"]
+                 [org.clojure/tools.logging "0.3.1"]
+                 [org.clojure/algo.generic "0.1.2"]
+                 [org.clojure/test.check "0.7.0"]
+                 [riemann-clojure-client  "0.3.2"]
+                 [overtone/at-at "1.2.0"]])

src/bigbrother/core.clj

@@ -0,0 +1,126 @@
+(ns bigbrother.core
+  "This file provide helpers to manage time spend in functions"
+  (:require [clojure.tools.logging :as log]
+            [clojure.data.json :as json]
+            [overtone.at-at :refer [every mk-pool]]
+            [riemann.client :as r]
+            [clojure.algo.generic.functor :refer [fmap]]
+
+            [bigbrother.timer :as timer]
+            [bigbrother.counter :as counter]
+            [bigbrother.metrics :as metrics]
+            [bigbrother.max-metrics :as max-metrics]
+            ))
+
+;; Atoms
+(def pool (atom nil)) ;; pool for async
+(def default-map (atom #{}))
+(def riemann-conn (atom nil))
+(def riemann-service (atom "supercell"))
+(def level-by-key (atom nil))
+
+(def n (atom 0))      ;; a number
+
+(def log-time timer/log-time)
+(def log-counter counter/log-counter)
+(def log-metric metrics/log-metric)
+(def log-mmetric max-metrics/log-mmetric)
+
+(defn timer-loop-finished []
+  ;; increment the number of loop
+  (swap! n inc)
+  (timer/finish-timer-loop)
+  (max-metrics/loop-finished)
+  (metrics/loop-finished)
+  (counter/loop-finished))
+
+;; ---- aliases
+(defn big-brother-is-watching-you "Starting the timer" []
+  (timer/log-time :start))
+(def telescreen-on "Starting the timer" big-brother-is-watching-you)
+
+(def welcome-in-miniluv "End the timer chrono" timer-loop-finished)
+(def telescreen-off "End the timer chrono" timer-loop-finished)
+
+;; ------------------------------------------------------------------------------
+;; Riemann Warn Level
+
+(defn warn-level
+  "warn level between two numbers"
+  [warn crit]
+  (fn [v] (cond (neg? v) "critical"
+                (>= v crit) "critical"
+                (>= v warn) "warning"
+                :else "ok")))
+(defn rev-warn-level
+  [warn crit]
+  (fn [v] (cond (<= v crit) "critical"
+                (<= v warn) "warning"
+                :else "ok")))
+
+(defn ok [] (fn [_] "ok"))
+(defn warning [] (fn [_] "warning"))
+(defn critical [] (fn [_] "critical"))
+
+(defn- to-riemann-event [[k v]]
+  (when (number? v)
+    (let [lvl-fn (get @level-by-key k)
+          level  (if lvl-fn (lvl-fn v) "ok")]
+      {:service (str @riemann-service " " (name k) (subs (str k) 1))
+       :state level 
+       :metric v})))
+
+(defn send-to-riemann [m]
+  (let [result-map (into @default-map m)
+        metric-data {:service @riemann-service
+                     :state "ok"}
+        events (remove nil? (map to-riemann-event result-map))
+        ]
+    (when @riemann-conn
+      (r/send-events @riemann-conn events))))
+
+
+(defn reset-accumulators! []
+  (counter/reset-acc!)
+  (metrics/reset-acc!)
+  (max-metrics/reset-acc!)
+  (timer/reset-acc!)
+  (reset! n 0))
+
+(defn reset-all-atoms! []
+  (reset! timer/times [])
+  (reset! metrics/metrics {})
+  (reset! max-metrics/mmetrics {})
+  (reset! counter/counters {})
+  (reset-accumulators!))
+
+(defn resume-map [nb-ms]
+  (let [n-by-sec  (float (/ @n (/ nb-ms 1000)))
+        basic     {:nb n-by-sec}]
+    (reduce into basic [(metrics/resume @n)
+                        (max-metrics/resume)
+                        (timer/resume)
+                        (counter/resume nb-ms)])))
+
+(defn display-time
+  "display the time at most every 10s"
+  [nb-ms]
+  (let [result (resume-map nb-ms)]
+    (log/info (json/write-str result))
+    (send-to-riemann result)
+    (reset-accumulators!)))
+
+(defn init-metrics
+  "## init-metrics
+
+    init-map :: Map Keyword (v -> ERROR_LEVEL)"
+  [init-map nb-ms-metrics riemann-host riemann-service-name]
+  (reset! default-map (reduce into {}
+                              (map (fn [k] {k -1} )
+                                   (conj (keys init-map) :total))))
+  (reset! level-by-key init-map)
+  (reset! pool (mk-pool))
+  (reset! riemann-service riemann-service-name)
+  (when riemann-host
+    (reset! riemann-conn (r/tcp-client {:host riemann-host})))
+  (every nb-ms-metrics (fn [] (display-time nb-ms-metrics)) @pool))

src/bigbrother/counter.clj

@@ -0,0 +1,39 @@
+(ns bigbrother.counter
+  (:require
+   [clojure.algo.generic.functor :refer [fmap]]))
+
+;; ------------------------------------------------------------------------------
+;; COUNTERS
+
+;; ## Monoid instance of `sumcounter`
+;; a sum time is a list of couple `{name nb}`
+(def empty-sumcounter {})
+(defn add-sumcounter [st st2]
+  (merge-with + st st2))
+
+;; Counters atoms
+(def counters (atom {}))
+(def sumcounters (atom empty-sumcounter))
+
+;; Counters (the mean is given by default)
+(defn- set-counter! [k v] (swap! counters add-sumcounter {k v}))
+(defn log-counter
+  "declare a specific counters (not time)"
+  ([k] (set-counter! k 1))
+  ([k v] (set-counter! k v)))
+(defn log-counter->
+  "declare a specific counters (not time) and returns the first parameter"
+  [x k v]
+  (log-counter k v)
+  x)
+
+(defn loop-finished []
+  ;; aggreate sumcounter
+  (swap! sumcounters add-sumcounter @counters)
+  (reset! counters {}))
+
+(defn reset-acc! []
+  (reset! sumcounters empty-sumcounter))
+
+(defn resume [nb-ms]
+  (fmap #(float (/ % (/ nb-ms 1000))) @sumcounters))

src/bigbrother/max_metrics.clj

@@ -0,0 +1,35 @@
+(ns bigbrother.max-metrics)
+
+;; ------------------------------------------------------------------------------
+;; Max Metrics
+
+;; ## Monoid instance of `maxmetrics`
+;; a sum time is a list of couple `{name metric-value}`
+(def empty-maxmetric {})
+(defn add-maxmetrics [st st2]
+  (merge-with max st st2))
+
+(def mmetrics (atom {})) ;; timestamps
+(def maxmetrics (atom empty-maxmetric)) ;; time spent by type
+
+;; Max Metrics (metrics to do a max between them instead of a mean)
+(defn- set-mmetric! [k v] (swap! mmetrics add-maxmetrics {k v}))
+(defn log-mmetric
+  "declare a specific max metrics (not time)"
+  [k v]
+  (set-mmetric! k v))
+(defn log-mmetric->
+  "declare a specific max metrics (not time) and returns the first parameter"
+  [x k v]
+  (log-mmetric k v)
+  x)
+
+(defn loop-finished []
+  ;; aggreate maxmetrics
+  (swap! maxmetrics add-maxmetrics @mmetrics)
+  (reset! mmetrics {}))
+
+(defn reset-acc! []
+  (reset! maxmetrics empty-maxmetric))
+
+(defn resume [] @maxmetrics)

src/bigbrother/metrics.clj

@@ -0,0 +1,37 @@
+(ns bigbrother.metrics
+  (:require [clojure.algo.generic.functor :refer [fmap]]))
+
+;; ------------------------------------------------------------------------------
+;; Sum Metrics
+
+;; ## Monoid instance of `summetrics`
+;; a sum time is a list of couple `{name metric-value}`
+(def empty-summetric {})
+(defn add-summetrics [st st2]
+  (merge-with + st st2))
+
+(def metrics (atom {})) ;; timestamps
+(def summetrics (atom empty-summetric)) ;; time spent by type
+
+;; Metrics (the mean is given by default)
+(defn- set-metric! [k v] (swap! metrics add-summetrics {k v}))
+(defn log-metric
+  "declare a specific metrics (not time)"
+  [k v]
+  (set-metric! k v))
+(defn log-metric->
+  "declare a specific metrics (not time) and returns the first parameter"
+  [x k v]
+  (log-metric k v)
+  x)
+
+(defn loop-finished []
+  ;; aggreate summetrics
+  (swap! summetrics add-summetrics @metrics)
+  (reset! metrics {}))
+
+(defn reset-acc! []
+  (reset! summetrics empty-summetric))
+
+(defn resume [n]
+  (fmap #(float (/ % n)) @summetrics))

src/bigbrother/timer.clj

@@ -0,0 +1,86 @@
+(ns bigbrother.timer)
+
+;; ------------------------------------------------------------------------------
+;; TIMERS
+
+;; ## Monoid instance of `sumtimes`
+;; a sum time is a list of couple `[name [timespent nb]]`
+(defn ts-name [x] (first x))
+(defn ts-timespent [x] (first (second x)))
+(defn ts-nb [x] (second (second x)))
+
+(def empty-sumtime [])
+(defn- add-one-sumtime [st st2]
+  [(ts-name st) [(+ (ts-timespent st) (ts-timespent st2))
+                 (+ (ts-nb st) (ts-nb st2))]])
+(defn add-sumtimes [st st2]
+  (cond (empty? st) st2
+        (empty? st2) st
+        :else (map add-one-sumtime st st2)))
+(defn fmap-sumtimes [f st]
+  (map (fn [v] [(first v) [(f (ts-timespent v))
+                           (ts-nb v)]]) st))
+
+(defn normalized-map-from-sumtimes
+  "Given a `sumtimes` returns a map `{key timespent}`'"
+  [st]
+  (reduce #(merge-with + %1 %2) {}
+          (map (fn [v] {(ts-name v)
+                        (/ (ts-timespent v)
+                           (ts-nb v))}) st)))
+;; timers atoms
+(def times (atom [])) ;; timestamps
+(def sumtimes (atom empty-sumtime)) ;; time spent by type
+
+;; Timer
+(defn- set-value! [k v] (swap! times conj [k v]))
+(defn log-time
+  "declare the action named `k` finished"
+  [k]
+  (set-value! k (System/nanoTime)))
+(defn log-time->
+  "declare the action named `k` finished and returned object `x`"
+  [x k]
+  (log-time k)
+  x)
+
+(defn- show-one-step
+  "get time spent during one step"
+  [x]
+  (if (< (count x) 2)
+    {:nothing 0}
+    (let [from (second (first x))
+          k    (first  (second x))
+          to   (second (second x))]
+      [k [(- to from) 1]])))
+
+(defn timespent
+  "from a list of timestamp generate a sumtime"
+  [times-array]
+  (map show-one-step (partition 2 1 times-array)))
+
+(defn total-time
+  "from a list of timestamp generate the total time spent"
+  [times-array]
+  (- (second (last times-array))
+     (second (first times-array))))
+
+(defn to-milliseconds
+  "convert from nanoseconds to milliseconds"
+  [times-array]
+  (fmap-sumtimes #(float (/ % 1000000)) times-array))
+
+(defn finish-timer-loop []
+  ;; Convert actual timestamps to sumtimes and aggregate them
+  (if (> (count @times) 1)
+    (let [difftime (timespent @times)
+          total    (total-time @times)
+          res      (to-milliseconds (conj difftime [:total [total 1]]))]
+      (swap! sumtimes add-sumtimes res)))
+  (reset! times []))
+
+(defn reset-acc! []
+  (reset! sumtimes empty-sumtime))
+
+(defn resume []
+  (normalized-map-from-sumtimes @sumtimes))

test/bigbrother/core_test.clj

@@ -0,0 +1,120 @@
+(ns bigbrother.core-test
+  (:require
+   [bigbrother.core :refer :all]
+
+   [clojure.test :refer :all]
+   [clojure.test.check.generators :as gen]
+   [clojure.test.check.properties :as prop]
+   [clojure.test.check.clojure-test :refer [defspec]]))
+
+(deftest timer-test
+  (do
+    (reset-all-atoms!)
+    ;; first loop
+    (telescreen-on)
+    (Thread/sleep 30)
+    (log-time :x1)
+    (Thread/sleep 30)
+    (log-time :x2)
+    (Thread/sleep 30)
+    (log-time :end)
+    (telescreen-off)
+    ;; second loop
+    (log-time :start)
+    (Thread/sleep 30)
+    (log-time :x2)
+    (Thread/sleep 30)
+    (log-time :end)
+    (telescreen-off)
+    (let [result (resume-map 1000)]
+      (is (contains? result :nb))
+      (is (contains? result :x1))
+      (is (contains? result :x2))
+      (is (>= (:total result) (:x2 result))))))
+
+(deftest check-metrics
+  (do
+    (reset-all-atoms!)
+    ;; first loop
+    (log-metric :foo 1)
+    (log-metric :bar 3)
+    (timer-loop-finished)
+    (log-metric :foo 2)
+    (log-metric :bar 2)
+    (timer-loop-finished)
+    (log-metric :foo 3)
+    (log-metric :bar 1)
+    (timer-loop-finished)
+    (let [result (resume-map 1000)]
+      (is (contains? result :foo))
+      (is (contains? result :bar))
+      (is (contains? result :nb))
+      (is (= 2.0 (:foo result)))
+      (is (= 2.0 (:bar result)))
+      (is (= 3.0 (:nb result))))))
+
+(deftest check-mmetrics
+  (do
+    (reset-all-atoms!)
+    (log-mmetric :foo 80)
+    (log-mmetric :bar 4)
+    (timer-loop-finished)
+    (log-mmetric :foo 50)
+    (log-mmetric :bar 5)
+    (timer-loop-finished)
+    (log-mmetric :foo 60)
+    (log-mmetric :bar 6)
+    (timer-loop-finished)
+    (let [result (resume-map 1000)]
+      (is (contains? result :foo))
+      (is (contains? result :bar))
+      (is (contains? result :nb))
+      (is (= 80 (:foo result)))
+      (is (= 6  (:bar result)))
+      (is (= 3.0 (:nb result))))))
+
+(deftest check-counters
+  (do
+    (reset-all-atoms!)
+    ;; first loop
+    (log-counter :foo)
+    (log-counter :foo)
+    (log-counter :bar 3)
+    (timer-loop-finished)
+    (log-counter :foo)
+    (log-counter :bar 2)
+    (timer-loop-finished)
+    (log-counter :foo)
+    (log-counter :bar 1)
+    (timer-loop-finished)
+    (let [result (resume-map 1000)]
+      (is (contains? result :foo))
+      (is (= 4.0 (:foo result)))
+      (is (= 6.0 (:bar result)))
+      (is (= 3.0 (:nb result))))))
+
+(defspec check-all-keys-exists
+  30
+  (prop/for-all
+   [actions (gen/such-that #(> (count %) 1)
+                           (gen/vector gen/keyword))]
+   (do
+     (reset-all-atoms!)
+     (doall (map (fn [a] (do (log-time a) (Thread/sleep 10)))
+                 actions))
+     (timer-loop-finished))
+   (let [result (resume-map 1000)]
+     (every? #(contains? result %) (rest actions)))))
+
+(defspec check-total-is-greater-than-each-input
+  30
+  (prop/for-all
+   [actions (gen/such-that #(> (count %) 1)
+                           (gen/vector gen/keyword))]
+   (do
+     (reset-all-atoms!)
+     (doall (map (fn [a] (do (log-time a) (Thread/sleep 10)))
+                 actions))
+     (timer-loop-finished))
+   (let [result (resume-map 1000)]
+     (every? #(>= (:total result) (get result %)) (rest actions)))))