updates & refactorings to how benchmarks are run. spit out nice results

src/clojure_template_benchmarks/core.clj

@@ -1,6 +1,7 @@
 (ns clojure-template-benchmarks.core
   (:use criterium.core
-        tinsel.core)
+        tinsel.core
+        clojure-csv.core)
   (:require [clabango.parser :refer [render render-file]]
             [selmer.parser :as selmer]
             [stencil.core :as stencil]
@@ -8,7 +9,7 @@
             [me.raynes.laser :as laser :refer [defdocument]]
             [net.cgrand.enlive-html :as enlive]
             [me.shenfeng.mustache :as mustache]
-            [selmer.parser :as selmer]))
+            [clj-pebble.core :as pebble]))
 
 (def bar (str "bar"))
 
@@ -41,7 +42,7 @@
 
 
 (defn simple-clabango-no-fd []
-  (render (str "<span class=\"foo\">{{bar}}</span>") {:bar bar}))
+  (render "<span class=\"foo\">{{bar}}</span>" {:bar bar}))
 
 (defn list-clabango-no-fd [ceil]
   (render "<ul>{% for item in items %}<li>{{item}}</li>{% endfor %}</ul>" {:items (range 1 ceil)}))
@@ -61,6 +62,27 @@
   (selmer/render-file "clojure_template_benchmarks/templates/list.html" {:items (range 1 ceil)}))
 
 
+(defn simple-selmer-no-fd []
+  (selmer/render "<span class=\"foo\">{{bar}}</span>" {:bar bar}))
+
+(defn list-selmer-no-fd [ceil]
+  (selmer/render "<ul>{% for item in items %}<li>{{item}}</li>{% endfor %}</ul>" {:items (range 1 ceil)}))
+
+
+(defn simple-pebble []
+  (pebble/render-file "src/clojure_template_benchmarks/templates/simple.html" {:bar bar}))
+
+(defn list-pebble [ceil]
+  (pebble/render-file "src/clojure_template_benchmarks/templates/list.html" {:items (range 1 ceil)}))
+
+
+(defn simple-pebble-no-fd []
+  (pebble/render "<span class=\"foo\">{{bar}}</span>" {:bar bar}))
+
+(defn list-pebble-no-fd [ceil]
+  (pebble/render "<ul>{% for item in items %}<li>{{item}}</li>{% endfor %}</ul>" {:items (range 1 ceil)}))
+
+
 (defn simple-stencil-no-fd []
   (stencil/render-string "<span class=\"foo\">{{bar}}</span>" {:bar bar}))
 
@@ -93,9 +115,9 @@
                                     (for [x (range 1 ceil)]
                                       (laser/node :li :content (str x)))))
 
-(defdocument simple-laser-hinted "<span class=\"foo\"></span>" []
+#_(defdocument simple-laser-hinted "<span class=\"foo\"></span>" []
              (laser/class= "foo") (laser/content ^String bar))
-(defdocument list-laser-hinted "<ul></ul>" [ceil]
+#_(defdocument list-laser-hinted "<ul></ul>" [ceil]
              (laser/element= :ul) (laser/content
                                     (for [x (range 1 ceil)]
                                       (laser/node :li :content (str ^Number x)))))
@@ -109,127 +131,200 @@
 (defn simple-enlive [] (apply str (simple-enlive-core)))
 (defn list-enlive [ceil] (apply str (list-enlive-core ceil)))
 
+(defmacro do-benchmarks
+  [benchmark-name simple-expr small-list-expr big-list-expr]
+  `(do
+     (println "\n **** Running Benchmark:" ~benchmark-name "*** \n")
+     (-> {}
+         (assoc
+           :simple
+           (do
+             (println ">>> Simple Data Injection\n")
+             (quick-benchmark ~simple-expr {})))
+         (assoc
+           :small-list
+           (do
+             (println ">>> Small List (50 items)\n")
+             (quick-benchmark ~small-list-expr {})))
+         (assoc
+           :big-list
+           (do
+             (println ">>> Big List (1000 items)\n")
+             (quick-benchmark ~big-list-expr {})))
+         (assoc :name ~benchmark-name))))
+
 (defn str-benches []
-  (println "\n\n ***** str benchmarks ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-str)))
+    "String"
-  (println "\n --- \n")
+    (simple-str)
-  (with-progress-reporting (quick-bench (list-str 50)))
+    (list-str 50)
-  (println "\n --- \n")
+    (list-str 1000)))
-  (with-progress-reporting (quick-bench (list-str 1000)))
-  (println "\n --- \n"))
 
 (defn hiccup-benches []
-  (println "\n\n ***** hiccup benchmarks  ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-hiccup)))
+    "Hiccup"
-  (println "\n --- \n")
+    (simple-hiccup)
-  (with-progress-reporting (quick-bench (list-hiccup 50)))
+    (list-hiccup 50)
-  (println "\n --- \n")
+    (list-hiccup 1000)))
-  (with-progress-reporting (quick-bench (list-hiccup 1000)))
+
-  (println "\n --- \n")
+(defn hiccup-benches-hinted []
-  (println "\n\n ***** type-hinted hiccup benchmarks  ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-hiccup-hint)))
+    "Hiccup (type-hinted)"
-  (println "\n --- \n")
+    (simple-hiccup-hint)
-  (with-progress-reporting (quick-bench (list-hiccup-hint 50)))
+    (list-hiccup-hint 50)
-  (println "\n --- \n")
+    (list-hiccup-hint 1000)))
-  (with-progress-reporting (quick-bench (list-hiccup-hint 1000)))
-  (println "\n --- \n"))
 
 (defn clabango-benches []
-  (println "\n\n ***** clabango string ***** \n\n")
+  (do-benchmarks
-  (quick-bench (simple-clabango-no-fd))
+    "Clabango"
-  (println "\n --- \n")
+    (simple-clabango-no-fd)
-  (quick-bench (list-clabango-no-fd 50))
+    (list-clabango-no-fd 50)
-  (println "\n --- \n")
+    (list-clabango-no-fd 1000)))
-  (quick-bench (list-clabango-no-fd 1000))
-  (println "\n --- \n")
 
-  (println "\n\n ***** clabango from file template ***** \n\n")
+
-  (with-progress-reporting (quick-bench (simple-clabango)))
+(defn clabango-benches-file []
-  (println "\n --- \n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (list-clabango 50)))
+    "Clabango (file)"
-  (println "\n --- \n")
+    (simple-clabango)
-  (with-progress-reporting (quick-bench (list-clabango 1000)))
+    (list-clabango 50)
-  (println "\n --- \n"))
+    (list-clabango 1000)))
 
 (defn selmer-benches []
-  (println "\n\n ***** selmer from file template ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-selmer)))
+    "Selmer"
-  (println "\n --- \n")
+    (simple-selmer-no-fd)
-  (with-progress-reporting (quick-bench (list-selmer 50)))
+    (list-selmer-no-fd 50)
-  (println "\n --- \n")
+    (list-selmer-no-fd 1000)))
-  (with-progress-reporting (quick-bench (list-selmer 1000)))
+
-  (println "\n --- \n"))
+(defn selmer-benches-file []
+  (do-benchmarks
+    "Selmer (file)"
+    (simple-selmer)
+    (list-selmer 50)
+    (list-selmer 1000)))
+
+(defn pebble-benches []
+  (do-benchmarks
+    "Pebble"
+    (simple-pebble-no-fd)
+    (list-pebble-no-fd 50)
+    (list-pebble-no-fd 1000)))
+
+(defn pebble-benches-file []
+  (do-benchmarks
+    "Pebble (file)"
+    (simple-pebble)
+    (list-pebble 50)
+    (list-pebble 1000)))
 
 (defn stencil-benches []
-  (println "\n\n ***** stencil string ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-stencil-no-fd)))
+    "Stencil"
-  (println "\n --- \n")
+    (simple-stencil-no-fd)
-  (with-progress-reporting (quick-bench (list-stencil-no-fd 50)))
+    (list-stencil-no-fd 50)
-  (println "\n --- \n")
+    (list-stencil-no-fd 1000)))
-  (with-progress-reporting (quick-bench (list-stencil-no-fd 1000)))
-  (println "\n --- \n")
 
-  (println "\n\n ***** stencil file ***** \n\n")
+(defn stencil-benches-hinted []
-  (with-progress-reporting (quick-bench (simple-stencil)))
+  (do-benchmarks
-  (println "\n --- \n")
+    "Stencil (file)"
-  (with-progress-reporting (quick-bench (list-stencil 50)))
+    (simple-stencil)
-  (println "\n --- \n")
+    (list-stencil 50)
-  (with-progress-reporting (quick-bench (list-stencil 1000)))
+    (list-stencil 1000)))
-  (println "\n --- \n"))
 
 (defn mustache-benches []
-  (println "\n\n ***** mustache file ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-mustache)))
+    "Mustache"
-  (println "\n --- \n")
+    (simple-mustache)
-  (with-progress-reporting (quick-bench (list-mustache 50)))
+    (list-mustache 50)
-  (println "\n --- \n")
+    (list-mustache 1000)))
-  (with-progress-reporting (quick-bench (list-mustache 1000)))
-  (println "\n --- \n"))
 
 (defn tinsel-benches []
-  (println "\n\n ***** tinsel ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-tinsel)))
+    "Tinsel"
-  (println "\n --- \n")
+    (simple-tinsel)
-  (with-progress-reporting (quick-bench (list-tinsel 50)))
+    (list-tinsel 50)
-  (println "\n --- \n")
+    (list-tinsel 1000)))
-  (with-progress-reporting (quick-bench (list-tinsel 1000)))
-  (println "\n --- \n"))
 
-(defn laser-benches []
+#_(defn laser-benches []
-  (println "\n\n ***** laser ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-laser)))
+    "Laser"
-  (println "\n --- \n")
+    (simple-laser)
-  (with-progress-reporting (quick-bench (list-laser 50)))
+    (list-laser 50)
-  (println "\n --- \n")
+    (list-laser 1000)))
-  (with-progress-reporting (quick-bench (list-laser 1000)))
-  (println "\n --- \n")
 
-  (println "\n\n ***** laser (hinted) ***** \n\n")
+#_(defn laser-benches-hinted []
-  (with-progress-reporting (quick-bench (simple-laser-hinted)))
+  (do-benchmarks
-  (println "\n --- \n")
+    "Laser (type-hinted)"
-  (with-progress-reporting (quick-bench (list-laser-hinted 50)))
+    (simple-laser-hinted)
-  (println "\n --- \n")
+    (list-laser-hinted 50)
-  (with-progress-reporting (quick-bench (list-laser-hinted 1000))))
+    (list-laser-hinted 1000)))
 
 (defn enlive-benches []
-  (println "\n\n ***** enlive ***** \n\n")
+  (do-benchmarks
-  (with-progress-reporting (quick-bench (simple-enlive)))
+    "Enlive"
-  (println "\n --- \n")
+    (simple-enlive)
-  (with-progress-reporting (quick-bench (list-enlive 50)))
+    (list-enlive 50)
-  (println "\n --- \n")
+    (list-enlive 1000)))
-  (with-progress-reporting (quick-bench (list-enlive 1000)))
-  (println "\n --- \n"))
 
-(defn -main [& args]
+(defn run-benchmarks []
-  ;; (println (simple-hiccup))
+  (doall
-  ;; (println (simple-clabango-no-fd))
+    (conj
-  ;; (println (count (list-filler-hiccup)))
+      []
-  ;; (println (count (list-filler-clabango-no-fd)))
       (selmer-benches)
+      (selmer-benches-file)
+      (pebble-benches)
+      (pebble-benches-file)
       (mustache-benches)
       (stencil-benches)
+      (stencil-benches-hinted)
       (str-benches)
       (hiccup-benches)
+      (hiccup-benches-hinted)
       (clabango-benches)
-  (laser-benches)
+      (clabango-benches-file)
-  (enlive-benches))
+      #_(laser-benches)
+      #_(laser-benches-hinted)
+      (enlive-benches)
+      )))
+
+(defn to-microsecs [t]
+  (* t 1000000))
+
+(defn get-std-dev [variance]
+  (to-microsecs (Math/sqrt variance)))
+
+(defn simplify-results [results]
+  (->> results
+       (map
+         (fn [{:keys [simple small-list big-list] :as result}]
+           (assoc result
+                  :simple (to-microsecs (first (:mean simple)))
+                  :simple-std-dev (get-std-dev (first (:variance simple)))
+                  :small-list (to-microsecs (first (:mean small-list)))
+                  :small-list-std-dev (get-std-dev (first (:variance small-list)))
+                  :big-list (to-microsecs (first (:mean big-list)))
+                  :big-list-std-dev (get-std-dev (first (:variance big-list)))
+                  )))
+       (sort-by :name)))
+
+(defn to-csv [simplified-results]
+  (map
+    (fn [{:keys [name simple simple-std-dev small-list small-list-std-dev big-list big-list-std-dev]}]
+      [name
+       (str simple)
+       (str simple-std-dev)
+       (str small-list)
+       (str small-list-std-dev)
+       (str big-list)
+       (str big-list-std-dev)])
+    simplified-results))
+
+(defn -main [& args]
+  (let [results (run-benchmarks)
+        results-table (simplify-results results)]
+    (clojure.pprint/print-table [:name :simple :simple-std-dev :small-list :small-list-std-dev :big-list :big-list-std-dev] results-table)
+    (as-> results-table x
+          (to-csv x)
+          (write-csv x :force-quote true)
+          (spit "results.csv" x))))
+