whoops, remove printlns

whoops, remove printlns

--- src/clj_http/core.clj
+++ src/clj_http/core.clj
@@ -179,12 +179,7 @@
                                         (DefaultHttpClient. conn-mgr))
         scheme (name scheme)]
     (when-let [cookie-store (or cookie-store *cookie-store*)]
-      (println :count? (count (.getCookies cookie-store)))
-      (and
-       (pos? (count (.getCookies cookie-store)))
-       (do
-         (println :adding-cs)
-         (.setCookieStore http-client cookie-store))))
+      (.setCookieStore http-client cookie-store))
     (when retry-handler
       (.setHttpRequestRetryHandler
        http-client


Simplify event listening

Simplify event listening

--- src/cljs/app/app.cljs
+++ src/cljs/app/app.cljs
@@ -7,7 +7,7 @@
             [kioo.reagent :as kioo]
             [cljs-http.client :as http])
   (:require-macros [kioo.reagent :refer [defsnippet deftemplate]]
-                   [cljs.core.async.macros :refer [go-loop]]))
+                   [cljs.core.async.macros :refer [go go-loop]]))

 ; ElasticSearch API URL
 (def query-url "http://168.235.155.245/beers/2015/_search")
@@ -52,25 +52,6 @@
 (def extract-hits-out (extract-hits get-results-out))
 (update-results-state extract-hits-out)

-; Use channels instead of callbacks for DOM interaction. Taken from
-; David Nolen's ClojureScript 101:
-; http://swannodette.github.io/2013/11/07/clojurescript-101/
-(defn listen [el type]
-  (let [out (chan)]
-    (events/listen el type
-      (fn [e] (put! out e)))
-    out))
-
-(defn search-field []
-  (dom/getElement "search-field"))
-
-; Capture keypresses on search field and perform searches as user types
-(defn query-input-loop []
-  (let [typing (listen (search-field) "keypress")]
-    (go-loop []
-      (<! typing)
-      (>! query-chan (.-value (search-field))))))
-
 ; Templating
 (def labels
   {:category-drugs "Therapeutic Category: Drug(s)"
@@ -92,11 +73,16 @@

 (deftemplate result-cards "templates/results.html" []
   {[:.card] (kioo/content (map result-card
-                                   (get-in @state [:results :_source])}))})
+                            (get-in @state [:results :_source])))})

 (deftemplate page "index.html" []
-  {[:.results] (kioo/content (result-cards))})
+  {[:.search-field] (kioo/listen :on-key-press
+                      (fn []
+                        (go
+                          (.log js/console "Press")
+                          (>! query-chan (.-value (dom/getElement "search-field"))))))
+
+   [:.results] (kioo/content (result-cards))})

 (defn init []
-  (reagent/render-component [page] (.-body js/document))
-  (query-input-loop))
+  (reagent/render-component [page] (.-body js/document)))


Utilize go-loop

Utilize go-loop

--- src/cljs/ji/main.cljs
+++ src/cljs/ji/main.cljs
@@ -20,7 +20,7 @@
             [dommy.template :refer [html->nodes]])
   (:require-macros
     [dommy.macros :refer [sel sel1 deftemplate node]]
-    [cljs.core.async.macros :refer [go alt!]]
+    [cljs.core.async.macros :refer [go go-loop alt!]]
     [cljs.core.match.macros :refer [match]]))

 (def heartbeat-interval 7500)
@@ -64,24 +64,24 @@
   "todo: general partition/chunking buffer"
   ([card-chan] (card-selector (chan) card-chan))
   ([c card-chan]
-   (go (loop [cs #{}]
-         (if (>= (count cs) 3)
-           (do (>! c cs) (recur #{}))
-           (let [v (<! card-chan)]
-             (cond
-               (nil? v) (close! c)
-               (cs v) (recur (disj cs v))
-               :else (recur (conj cs v)))))))
+   (go-loop [cs #{}]
+     (if (>= (count cs) 3)
+       (do (>! c cs) (recur #{}))
+       (let [v (<! card-chan)]
+         (cond
+           (nil? v) (close! c)
+           (cs v) (recur (disj cs v))
+           :else (recur (conj cs v))))))
    c))

 (defn go-emit-selections [out card-sel]
   (let [sels (card-selector card-sel)]
-    (go (loop []
-          (when-let [cards (<! sels)]
-            (>! out (msg/->PlayerSetMessage cards))
-            (doseq [el (sel [:#board :.selected])]
-              (dom/remove-class! el "selected"))
-            (recur))))))
+    (go-loop []
+      (when-let [cards (<! sels)]
+        (>! out (msg/->PlayerSetMessage cards))
+        (doseq [el (sel [:#board :.selected])]
+          (dom/remove-class! el "selected"))
+        (recur)))))

 (defn render-solutions!
   [sets]
@@ -117,12 +117,12 @@
 (defn go-emit-heartbeat
   "A go loop to regularly write heartbeat message on channel"
   [control out interval]
-  (go (loop []
-        (let [tick (timeout interval)]
-          (match (alts! [control tick])
-            [nil control] nil
-            [_ tick] (do (>! out heartbeat-req)
-                         (recur)))))))
+  (go-loop []
+    (let [tick (timeout interval)]
+      (match (alts! [control tick])
+        [nil control] nil
+        [_ tick] (do (>! out heartbeat-req)
+                     (recur))))))

 (defn go-game [container in out player-id]
   (let [control (chan) ;; TODO integrate further
@@ -144,37 +144,37 @@
                        (sel1 container :#game-status))

     ;; main game loop
-    (go (loop []
-          (let [msg (<! in)]
-            (cond
-              (nil? msg)
-              (do
-                (close! control)
-                (close! card-sel)
-                (close! board-state)
-                (close! player-state)
-                (close! status-state)
-                (dom/set-html! container "")
-                (msg/error "Disconnected from server"))
-
-              (= msg heartbeat-resp)
-              (do (recur))
-
-              (instance? msg/GameStateMessage msg)
-              (do (go-game-state (:game msg) board-state player-state status-state)
-                (recur))
-
-              (instance? msg/GameFinishMessage msg)
-              (let [{:keys [game]} msg]
-                (>! board-state :disable)
-                (<! (go-game-summary container msg))
-                (close! board-state)
-                (close! player-state)
-                msg)
-
-              :else
-              (do (println "UNHANDLED MSG" msg)
-                  (recur))))))))
+    (go-loop []
+      (let [msg (<! in)]
+        (cond
+          (nil? msg)
+          (do
+            (close! control)
+            (close! card-sel)
+            (close! board-state)
+            (close! player-state)
+            (close! status-state)
+            (dom/set-html! container "")
+            (msg/error "Disconnected from server"))
+
+          (= msg heartbeat-resp)
+          (do (recur))
+
+          (instance? msg/GameStateMessage msg)
+          (do (go-game-state (:game msg) board-state player-state status-state)
+              (recur))
+
+          (instance? msg/GameFinishMessage msg)
+          (let [{:keys [game]} msg]
+            (>! board-state :disable)
+            (<! (go-game-summary container msg))
+            (close! board-state)
+            (close! player-state)
+            msg)
+
+          :else
+          (do (println "UNHANDLED MSG" msg)
+              (recur)))))))

 (defn run-game!
   [container game-id player-id]


Update space age test to cast to float before rounding.

--- space-age/space_age_test.clj
+++ space-age/space_age_test.clj
@@ -5,8 +5,8 @@

 (defn- rounds-to
   [expected actual]
-  (is (= (Math/round (* 100 expected))
-         (Math/round (* 100 actual)))))
+  (is (= (Math/round (* 100.0 expected))
+         (Math/round (* 100.0 actual)))))

 (deftest age-in-earth-years
   (rounds-to 31.69 (space-age/on-earth 1000000000)))


The one useful thing liberator did, duplicate that here.


Add support for dynamically set the byte order.

--- src/octet/buffer.cljc
+++ src/octet/buffer.cljc
@@ -1,4 +1,4 @@
-;; Copyright (c) 2015 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) 2015-2016 Andrey Antukh <niwi@niwi.nz>
 ;; All rights reserved.
 ;;
 ;; Redistribution and use in source and binary forms, with or without
@@ -34,8 +34,16 @@
   es6 typed arrays (from javascript environments)."
   #?(:clj
      (:import java.nio.ByteBuffer
+              java.nio.ByteOrder
               io.netty.buffer.ByteBuf
               io.netty.buffer.ByteBufAllocator)))
+
+(def ^:dynamic *byteorder*
+  "Defines the default byte order used for the write and read
+  operations on the byte buffers."
+  :big-endian)
+
+;; --- Protocols

 (defprotocol IBufferShort
   (read-short [_ pos] "Read short integer (16 bits) from buffer.")
@@ -76,70 +84,94 @@
 (defprotocol IBufferLimit
   (get-capacity [_] "Get the read/write capacity in bytes."))

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; NIO & Netty Buffer implementations
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; ---- NIO & Netty Buffer implementations
+
+(defn- set-current-bytebuffer-byteorder!
+  [buff]
+  (case *byteorder*
+    :big-endian (.order buff ByteOrder/BIG_ENDIAN)
+    :little-endian (.order buff ByteOrder/LITTLE_ENDIAN)))

 #?(:clj
    (extend-type ByteBuffer
      IBufferShort
      (read-short [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.getShort buff pos))
      (write-short [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.putShort buff pos value))
      (read-ushort [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (let [val (.getShort buff pos)]
          (bit-and 0xFFFF (Integer. (int val)))))
      (write-ushort [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (let [value (.shortValue (Integer. (int value)))]
-      (.putShort buff pos value)))
+         (.putShort buff pos value)))

      IBufferInt
      (read-int [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.getInt buff pos))
      (write-int [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.putInt buff pos value))
      (read-uint [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (let [val (.getInt buff pos)]
          (bit-and 0xFFFFFFFF (Long. (long val)))))
      (write-uint [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (let [value (.intValue (Long. (long value)))]
          (.putInt buff pos value)))

      IBufferLong
      (read-long [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.getLong buff pos))
      (write-long [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.putLong buff pos value))
      (read-ulong [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (let [val (.getLong buff pos)
              ^bytes magnitude (-> (ByteBuffer/allocate 8) (.putLong val) .array)]
          (bigint (BigInteger. 1 magnitude))))
      (write-ulong [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (let [value (.longValue (bigint value))]
          (.putLong buff pos value)))

      IBufferFloat
      (read-float [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.getFloat buff pos))
      (write-float [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.putFloat buff pos value))

      IBufferDouble
      (read-double [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.getDouble buff pos))
      (write-double [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.putDouble buff pos value))

      IBufferByte
      (read-byte [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (.get buff pos))
      (write-byte [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (.put buff pos value))
      (read-ubyte [buff pos]
+       (set-current-bytebuffer-byteorder! buff)
        (let [val (.get buff pos)]
          (bit-and 0xFF (short val))))
      (write-ubyte [buff pos value]
+       (set-current-bytebuffer-byteorder! buff)
        (let [value (.byteValue (Short. (short value)))]
          (.put buff pos value)))

@@ -165,52 +197,72 @@
    (extend-type ByteBuf
      IBufferShort
      (read-short [buff pos]
-       (.getShort buff pos))
+       (if (= *byteorder* :little-endian)
+         (.getShortLE buff pos)
+         (.getShort buff pos)))
      (write-short [buff pos value]
-       (.setShort buff pos value))
+       (if (= *byteorder* :little-endian)
+         (.setShortLE buff pos value)
+         (.setShort buff pos value)))
      (read-ushort [buff pos]
-       (let [val (.getShort buff pos)]
+       (let [val (read-short buff pos)]
          (bit-and 0xFFFF (Integer. (int val)))))
      (write-ushort [buff pos value]
        (let [value (.shortValue (Integer. (int value)))]
-         (.setShort buff pos value)))
+         (write-short buff pos value)))

      IBufferInt
      (read-int [buff pos]
-       (.getInt buff pos))
+       (if (= *byteorder* :little-endian)
+         (.getIntLE buff pos)
+         (.getInt buff pos)))
      (write-int [buff pos value]
-       (.setInt buff pos value))
+       (if (= *byteorder* :little-endian)
+         (.setIntLE buff pos value)
+         (.setInt buff pos value)))
      (read-uint [buff pos]
-       (let [val (.getInt buff pos)]
+       (let [val (read-int buff pos)]
          (bit-and 0xFFFFFFFF (Long. (long val)))))
      (write-uint [buff pos value]
        (let [value (.intValue (Long. (long value)))]
-         (.setInt buff pos value)))
+         (write-int buff pos value)))

      IBufferLong
      (read-long [buff pos]
-       (.getLong buff pos))
+       (if (= *byteorder* :little-endian)
+         (.getLongLE buff pos)
+         (.getLong buff pos)))
      (write-long [buff pos value]
-       (.setLong buff pos value))
+       (if (= *byteorder* :little-endian)
+         (.setLongLE buff pos value)
+         (.setLong buff pos value)))
      (read-ulong [buff pos]
-       (let [val (.getLong buff pos)
+       (let [val (read-long buff pos)
              ^bytes magnitude (-> (ByteBuffer/allocate 8) (.putLong val) .array)]
          (bigint (BigInteger. 1 magnitude))))
      (write-ulong [buff pos value]
        (let [value (.longValue (bigint value))]
-         (.setLong buff pos value)))
+         (write-long buff pos value)))

      IBufferFloat
      (read-float [buff pos]
-       (.getFloat buff pos))
+       (if (= *byteorder* :little-endian)
+         (.getFloatLE buff pos)
+         (.getFloat buff pos)))
      (write-float [buff pos value]
-       (.setFloat buff pos value))
+       (if (= *byteorder* :little-endian)
+         (.setFloatLE buff pos value)
+         (.setFloat buff pos value)))

      IBufferDouble
      (read-double [buff pos]
-       (.getDouble buff pos))
+       (if (= *byteorder* :little-endian)
+         (.getDoubleLE buff pos)
+         (.getDouble buff pos)))
      (write-double [buff pos value]
-       (.setDouble buff pos value))
+       (if (= *byteorder* :little-endian)
+         (.setDoubleLE buff pos value)
+         (.setDouble buff pos value)))

      IBufferByte
      (read-byte [buff pos]
@@ -236,9 +288,7 @@
      (get-capacity [buff]
        (.capacity buff))))

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; A vector of buffers
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; --- A vector of buffers

 (defn- stream-operation
   [opfn buff pos]
@@ -335,43 +385,41 @@
   (get-capacity [buff]
     (reduce #(+ %1 (get-capacity %2)) 0 buff)))

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; ES6 Typed Arrays
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; --- ES6 Typed Arrays

 #?(:cljs
    (extend-type js/DataView
      IBufferShort
      (read-short [buff pos]
-       (.getInt16 buff pos))
+       (.getInt16 buff pos (= *byteorder* :little-endian)))
      (write-short [buff pos value]
-       (.setInt16 buff pos value))
+       (.setInt16 buff pos value (= *byteorder* :little-endian)))
      (read-ushort [buff pos]
-       (.getUint16 buff pos))
+       (.getUint16 buff pos (= *byteorder* :little-endian)))
      (write-ushort [buff pos value]
-       (.setUint16 buff pos value))
+       (.setUint16 buff pos value (= *byteorder* :little-endian)))

      IBufferInt
      (read-int [buff pos]
-       (.getInt32 buff pos))
+       (.getInt32 buff pos (= *byteorder* :little-endian)))
      (write-int [buff pos value]
-       (.setInt32 buff pos value))
+       (.setInt32 buff pos value (= *byteorder* :little-endian)))
      (read-uint [buff pos]
-       (.getUint32 buff pos))
+       (.getUint32 buff pos (= *byteorder* :little-endian)))
      (write-uint [buff pos value]
-       (.setUint32 buff pos value))
+       (.setUint32 buff pos value (= *byteorder* :little-endian)))

      IBufferFloat
      (read-float [buff pos]
-       (.getFloat32 buff pos))
+       (.getFloat32 buff pos (= *byteorder* :little-endian)))
      (write-float [buff pos value]
-       (.setFloat32 buff pos value))
+       (.setFloat32 buff pos value (= *byteorder* :little-endian)))

      IBufferDouble
      (read-double [buff pos]
-       (.getFloat64 buff pos))
+       (.getFloat64 buff pos (= *byteorder* :little-endian)))
      (write-double [buff pos value]
-       (.setFloat64 buff pos value))
+       (.setFloat64 buff pos value (= *byteorder* :little-endian)))

      IBufferByte
      (read-byte [buff pos]
@@ -396,13 +444,10 @@
      (get-capacity [buff]
        (.-byteLength buff))))

-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Public Api
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-#?(:clj
-   (def ^{:private true}
-     allocator (ByteBufAllocator/DEFAULT)))
+;; --- Public Api
+
+#?(:clj
+   (def ^:private allocator (ByteBufAllocator/DEFAULT)))

 (defmulti allocate
   "Polymorphic function for allocate new bytebuffers.
@@ -424,7 +469,7 @@

   This function is defined as multimethod and you can
   extend it with your own bytebuffer implementations
-  if you want or need."
+  if you want or need it."
   (fn [size & [{:keys [type impl] :or {type :heap
                                        impl #?(:clj :nio :cljs :es6)}}]]
     [type impl]))


github handler をデバッグ用に修正

github handler をデバッグ用に修正

--- src/panpan/github.clj
+++ src/panpan/github.clj
@@ -4,9 +4,10 @@

 (defn github-handler
   ""
-  [{:keys [user] :as arg}]
-  (when (or true (= "github" user))
-    (handler/regexp
+  [{:keys [user text] :as arg}]
+  (when (= "github" user)
+    (str "DEBUG:\"" text "\"")
+    #_(handler/regexp
       arg
       #"(?s)^\[(.+?)\].+Issue created.+(#\d+)"
       (fn [{[_ repo issue] :match}]


Correct depencency versions. The cljs-ajax change is a reprise of 443eaf9.:

--- metlog-vault/project.clj
+++ metlog-vault/project.clj
@@ -4,7 +4,7 @@
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}

-  :plugins [[lein-cljsbuild "1.0.6"
+  :plugins [[lein-cljsbuild "1.1.1"
              :exclusions [org.clojure/clojure]]
             [lein-cooper "1.1.1"]]

@@ -21,7 +21,8 @@
                  [hiccup "1.0.5"]

                  [org.clojure/core.async "0.2.374"]
-                 [cljs-ajax "0.5.3"]
+                 ;; Old version to work around https://github.com/JulianBirch/cljs-ajax/issues/109
+                 [cljs-ajax "0.3.11"]
                  [reagent "0.5.1"]
                  [com.andrewmcveigh/cljs-time "0.4.0"]


crawl framework

crawl framework

--- src/crawler/crawl.clj
+++ src/crawler/crawl.clj
@@ -1,12 +1,24 @@
 (ns crawler.crawl
   "Initial crawl setup"
   (:require [clj-http.client :as client]
-            [crawler.rich-extractor :as extractor]))
+            [clojure.java.io :as io]
+            [crawler.rich-extractor :as rich-extractor])
+  (:use [clojure.pprint :only [pprint]]))

 (defn write
-  [content]
-  (println :write (content :url)))
+  [content filename]
+  (spit filename (with-out-str (prn content)) :append true))
+
+(defn distinct-by-key
+ [coll k]
+ (reduce
+  (fn [acc v]
+    (if (some #{(v k)} (set (map #(k %) acc)))
+      acc
+      (cons v acc)))
+  []
+  coll))

 (defn crawl-richest
   "Main crawl routine
@@ -18,110 +30,209 @@
    Representation for content XPath is important. What kind
    of representation to use? Target-cluster?

-   Queue looks like: [{:url <url>
+   A queue looks like: [{:url <url>
                       :content-decision <content-xpath>
                       :pag-decision <pag-xpath>
                       :paginated?}
                       .
                       .
                       .]
-   misc: {:num-pages-to-max ..}. Number of pages to follow overall.
-   what is the base idea."
-  [queue visited num-decisions sum-score limit options]
+   We have pagination and content queues.
+   A decision looks like:
+
+   [XPath {:links []}]
+
+   globals: use this to store global info for the crawler"
+  [queues visited num-decisions sum-score limit globals]
   (Thread/sleep 2000)
+  (println globals)
   (if (and (not (zero? limit))
-           (seq queue))
-    (let [url        (-> queue first :url)
-          body       (-> url client/get :body)
-          paginated? (-> queue first :pagination?)
-          p-thus-far (-> options :pagination)
-          decision   (extractor/extract-above-average-richest
-                      body url (clojure.set/union visited queue))
-          xpaths     (:xpaths decision)
-          links      (:links decision)
-          score      (:action-score decision)]
+           (or (seq (-> queues :content))
+               (seq (-> queues :pagination))))
+    (let [content-q   (-> queues :content)         ; holds the content queues
+          paging-q    (-> queues :pagination)      ; holds the pagination queues
+
+          queue-kw    (cond
+                       (empty? content-q)
+                       :pagination
+
+                       (empty? paging-q)
+                       :content
+
+                       :else
+                       (if (even? (int (/ (count visited) 10)))
+                         :pagination :content))
+
+          queue       (queues queue-kw)
+
+          url         (-> queue first :url)
+          body        (-> url client/get :body)
+          new-visited (conj visited url)
+          paginated?  (-> queue first :pagination?)
+
+          ;; extract from whatever needs extracting
+          extracted   (rich-extractor/extract-above-average-richest
+                       body
+                       (first queue)
+                       (clojure.set/union visited queue))
+          decision    (:decision extracted)
+          xpaths      (map first decision)
+          score       (:score extracted)
+          leaf?       (rich-extractor/leaf? sum-score num-decisions score)]
       (do
         (println :total-score (/ sum-score num-decisions))
         (println :cur-page-score score)
-        (if (< (* 0.75 (/ sum-score num-decisions)) score)
-          (let [paging-decis (extractor/weak-pagination-detector
-                              body
-                              (first queue)
-                              (clojure.set/union visited queue))
-                pagination-l (second paging-decis)
-
-                new-queue    (if (<= (-> options :pagination) 10) ; switch order of insertion at 10 urls
-                               (do
-                                 (println :links links)
-                                 (println :pagination paging-decis)
-                                 (println)
-                                 (concat (map
-                                          (fn [a-link]
-                                            {:url a-link
-                                             :pagination? true
-                                             :content-xpaths xpaths
-                                             :paging-xpaths (first paging-decis)}) ; xpaths are what we picked on this page. no need to re-learn anything
-                                          pagination-l)
-                                         (rest queue)
+        (if (or paginated?
+                (not leaf?))
+          (do
+            (write
+             {:url       url
+              :body      body
+              :src-url   (-> queue first :source)
+              :src-xpath (-> queue first :src-xpath)}
+             "crawl.json")
+            (let [ ;; the decision made by the pagination component
+                  paging-dec   (rich-extractor/weak-pagination-detector
+                                body
+                                (first queue)
+                                globals
+                                (clojure.set/union visited
+                                                   (map #(-> % :url) content-q)
+                                                   (map #(-> % :url) paging-q)
+                                                   [url]))
+
+                  ;; pagination's xpath and links
+                  paging-xp-ls (into
+                                {} (map
+                                    (fn [[xpath info]]
+                                     [xpath (:links info)])
+                                    paging-dec))
+
+                  paging-vocab (reduce
+                                clojure.set/union
+                                (map
+                                 (fn [[xpath info]]
+                                   (:vocab info))
+                                 paging-dec))
+
+                  new-globals  (merge-with clojure.set/union
+                                           globals
+                                           {:paging-vocab paging-vocab})
+
+                  ;; add to content-q if anything needs adding/removing
+                  content-q'   (concat (if (= :content queue-kw)
+                                         (rest content-q) content-q)
+                                       (distinct-by-key
+                                        (flatten
                                          (map
-                                          (fn [x] {:url x})
-                                          (filter
-                                           (fn [a-link]
-                                             (and (not (some #{a-link} visited))
-                                                  (not (some #{a-link} (set
-                                                                        (map #(-> % :url) queue))))))
-                                           links))))
-                               (do
-                                 (println :links links)
-                                 (println :pagination paging-decis)
-                                 (println)
-                                 (concat
-                                  (rest queue)
-                                  (map
-                                   (fn [x] {:url x})
-                                   (filter
-                                    (fn [a-link]
-                                      (and (not (some #{a-link}
-                                                      visited))
-                                           (not (some #{a-link}
-                                                      (set
-                                                       (map #(-> % :url) queue))))))
-                                           links))
-                                  (map (fn [a-link]
-                                         {:url a-link
-                                          :pagination? true
-                                          :content-xpaths xpaths
-                                          :paging-xpaths (first paging-decis)}) ; xpaths are what we picked on this page. no need to re-learn anything
-                                       pagination-l))))
-                new-visited  (conj visited url)
-                new-num-dec  (inc num-decisions)
-                new-scr      (+ score sum-score)
-                new-lim      (dec limit)
-                new-opt      (if paginated?
-                               (merge
-                                options
-                                {:pagination (+ (-> options :pagination) 1)})
-                               (merge options {:pagination 0}))]
-            (recur new-queue
-                   new-visited
-                   new-num-dec
-                   new-scr
-                   new-lim
-                   new-opt))
+                                          (fn [[x links]]
+                                            (filter
+                                             identity
+                                             (map
+                                              (fn [a-link]
+                                                (when-not (and (some #{a-link} visited)
+                                                               (some #{a-link}
+                                                                     (set
+                                                                      (map #(-> % :url) content-q))))
+                                                  {:url a-link
+                                                   :source url
+                                                   :src-xpath x}))
+                                              links)))
+                                          decision))
+                                        :url))
+
+                  ;; add to paging-q if anything needs adding/removing
+                  paging-q'    (concat (if (= :pagination queue-kw)
+                                         (rest paging-q) paging-q)
+                                       (distinct-by-key
+                                        (flatten
+                                         (map (fn [[xpath links]]
+                                                (map
+                                                 (fn [a-link]
+                                                   (when-not (and (some #{a-link} visited)
+                                                                  (some #{a-link}
+                                                                        (set
+                                                                         (map #(-> % :url) content-q))))
+                                                     {:url a-link
+                                                      :source url
+                                                      :src-xpath xpath
+                                                      :pagination? true
+                                                      :content-xpaths xpaths}))
+                                                 links))
+                                              paging-xp-ls))
+                                        :url))
+
+
+                  new-num-dec  (inc num-decisions)
+                  new-scr      (+ score sum-score)
+                  new-lim      (dec limit)]
+              (recur {:content content-q'
+                      :pagination paging-q'}
+                     new-visited
+                     new-num-dec
+                     new-scr
+                     new-lim
+                     new-globals)))
           (do
             (println :no-links-chosen)
             (println)
-                                        ; pagination must still be chosen though
-            (let [new-queue   (rest queue)
-                  new-visited (conj visited url)
+            (write
+             {:url       url
+              :body      body
+              :src-url   (-> queue first :source)
+              :src-xpath (-> queue first :src-xpath)
+              :leaf?     true}
+             "crawl.json")
+            (let [paging-dec   (rich-extractor/weak-pagination-detector
+                                body
+                                (first queue)
+                                globals
+                                (clojure.set/union visited
+                                                   (map #(-> % :url) content-q)
+                                                   (map #(-> % :url) paging-q)
+                                                   [url]))
+
+                ;; pagination's xpath and links
+                  paging-xp-ls (into
+                                {} (map
+                                    (fn [[xpath info]]
+                                      [xpath (:links info)])
+                                    paging-dec))
+
+                  paging-vocab (reduce
+                                clojure.set/union
+                                (map
+                                 (fn [[xpath info]]
+                                   (:vocab info))
+                                 paging-dec))
+
+                  new-globals  (merge-with clojure.set/union
+                                         globals
+                                         {:paging-vocab paging-vocab})
+
+                  new-queue    (if (= :pagination queue-kw)
+                                 {:content content-q
+                                  :pagination (rest paging-q)}
+                                 {:content (rest content-q)
+                                  :pagination paging-q})
                   new-num-dec num-decisions
                   new-scr     sum-score
-                  new-lim     (dec limit)
-                  new-opt     options]
+                  new-lim     (dec limit)]
               (recur new-queue
                      new-visited
                      new-num-dec
                      new-scr
                      new-lim
-                     new-opt))))))
+                     new-globals))))))
     {:visited visited}))
+
+(defn crawl
+  [start crawler-type num-docs]
+  (cond (= :richness crawler-type)
+        (crawl-richest {:content [{:url start}]
+                        :pagination []}
+                       (set [])
+                       1
+                       0
+                       num-docs
+                       {})))


Rename resolve function to `join`.

Rename resolve function to `join`.

--- src/datoteka/core.clj
+++ src/datoteka/core.clj
@@ -1,4 +1,4 @@
-;; Copyright (c) 2015-2016 Andrey Antukh <niwi@niwi.nz>
+;; Copyright (c) 2015-2017 Andrey Antukh <niwi@niwi.nz>
 ;; All rights reserved.
 ;;
 ;; Redistribution and use in source and binary forms, with or without
@@ -24,7 +24,7 @@

 (ns datoteka.core
   "File System helpers."
-  (:refer-clojure :exclude [name resolve])
+  (:refer-clojure :exclude [name])
   (:require [datoteka.proto :as pt]
             [clojure.java.io :as io])
   (:import java.io.Writer
@@ -216,13 +216,15 @@

       :else (pt/-path path))))

-(defn resolve
+(defn join
   "Resolve path on top of an other path."
-  [base path]
-  (let [^Path base (pt/-path base)
-        ^Path path (pt/-path path)]
-    (-> (.resolve base path)
-        (.normalize))))
+  ([base path]
+   (let [^Path base (pt/-path base)
+         ^Path path (pt/-path path)]
+     (-> (.resolve base path)
+         (.normalize))))
+  ([base path & more]
+   (reduce join base (cons path more))))

 (defn relativize
   "Returns the relationship between two paths."


Use new Discuss view in core function

Use new Discuss view in core function

--- src/discuss/core.cljs
+++ src/discuss/core.cljs
@@ -19,7 +19,7 @@
 ;; Initialization
 (defn ^:export main []
   (log/debug "Loaded %s %s" config/project (lib/project-version))
-  (om/add-root! parser/reconciler views/MainView (gdom/getElement (lib/prefix-name "main")))
+  (om/add-root! parser/reconciler views/Discuss (gdom/getElement (lib/prefix-name "main")))
   (rint/request-references)
   (comlib/init!))
 (main)


Fixed small bug in parser

Fixed small bug in parser

--- src/selmer/parser.clj
+++ src/selmer/parser.clj
@@ -1,6 +1,7 @@
 (ns selmer.parser
   (:require [clojure.string :as s]
-            [selmer.filters :refer [get-filter]])
+            [selmer.filters :refer [get-filter]]
+            [selmer.tags :refer [tags]])
   (:import [java.io PushbackReader CharArrayReader]))

 ;;; TODO - implement filter/tag parsers
@@ -31,7 +32,9 @@
         (case next-step
           ;; Found end of string
           ;; TODO - error if there's still a closing tag to find
-          :end ast
+          :end (if (seq buffered-str)
+                 (conj ast buffered-str)
+                 ast)
           :read-filter (recur :read-filter
                               (conj ast buffered-str)
                               to-find)
@@ -285,8 +288,8 @@
   [{:keys [tag args body]}]
   (fn [context-map]
     (if args
-      (apply (get-tag tag) args body)
-      (apply (get-tag) body))))
+      (apply (get-tag tag) context-map args body)
+      (apply (get-tag) context-map body))))

 ;;; End of tag compiling


Clean up sample-jobdef.clj example

Clean up sample-jobdef.clj example

--- examples/sample-jobdef.clj
+++ examples/sample-jobdef.clj
@@ -150,21 +150,20 @@
   ; NO DEFAULT
   ;:runtime-jar "s3://com.your-co.${env}.hadoop/marc/my-hadoop-java.jar"

-  ;Number of instances (including the master)
+  ; Number of instances (including the master)
   ;:num-instances 1

-  ;Instance type for slaves
+  ; Instance type for slaves
   ;:slave-instance-type "m1.large"

-  ;Instance type for master
+  ; Instance type for master
   ;:master-instance-type "m1.large"

-  ;EMR Cluster tags
-  ;Provide a comma-separated list of key=value pairs, which will be used to
-  ;create tags for instances in your EMR cluster
-  ;read more at
-  ;http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-plan-tags.html
+  ; Provide a comma-separated list of key=value pairs, which will be used to
+  ; create tags for instances in your EMR cluster. Read more at
+  ; http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-plan-tags.html
   ;:tags "fake:project=fightingcrime,fake:group=ateam"
+
   ; Uncomment to attempt to get additional nodes via the spot market
   ; This example requests up to 30 additional m1.xlarge nodes, and we are willing to
   ; pay up to 80% of the difference between the reserve price and the demand price.


reformat for more conventional style

reformat for more conventional style

--- src/immutant/init.clj
+++ src/immutant/init.clj
@@ -1,12 +1,10 @@
 (ns immutant.init
-  (:require
-   [clojure.edn :as edn]
-   [kilo.web           :as k-web]
-   [kilo.sqldb         :as k-sqldb]
-   [kilo.messaging.listener :as k-listener]
-   [immutant.web       :as web]
-   [immutant.messaging :as msg]
-   ))
+  (:require [clojure.edn :as edn]
+            [kilo.web :as k-web]
+            [kilo.sqldb :as k-sqldb]
+            [kilo.messaging.listener :as k-listener]
+            [immutant.web :as web]
+            [immutant.messaging :as msg]))

 (defn initialize!
   []


+(rewrite-lexicon)

+(rewrite-lexicon)

--- src/babel/english/writer.clj
+++ src/babel/english/writer.clj
@@ -12,6 +12,9 @@
                    process write-lexicon reload]])
 (require '[clojure.tools.logging :as log])
 (require '[dag-unify.core :refer (fail? get-in strip-refs unify)])
+
+(defn rewrite-lexicon []
+  (write-lexicon "en" @lexicon))

 (defn expression [spec]
   (writer/expression small-plus-plus-np spec))


rename function

rename function

--- src/babel/espanol/writer.clj
+++ src/babel/espanol/writer.clj
@@ -30,7 +30,7 @@
 (defn fo [expression]
   (morph/fo expression))

-(defn reset []
+(defn reset! []
   (refresh-all)
   (log4j!))


need to return a channel

need to return a channel

--- src/bitsplit_cli/client.cljs
+++ src/bitsplit_cli/client.cljs
@@ -24,7 +24,8 @@
                     (put! return (amounts @index))
                     (swap! index inc)
                     (swap! index #(mod % 3)))
-                1000)))
+                1000)
+            return))
     Operations
     (send-amounts! [this amounts])
     (new-address! [this]))


Hide header on non-group pages

Hide header on non-group pages

--- src/braid/ui/views/main.cljs
+++ src/braid/ui/views/main.cljs
@@ -35,8 +35,11 @@
             [:h1 "???"])))))

 (defn main-view []
-  [:div.main
-   [error-banner-view]
-   [sidebar-view]
-   [header-view]
-   [page-view]])
+  (let [group-id (subscribe [:open-group-id])]
+    (fn []
+      [:div.main
+       [error-banner-view]
+       [sidebar-view]
+       (when @group-id
+         [header-view])
+       [page-view]])))


index-directory: remove some unused :requires and rename mapping*to name->smanager*

--- src/bzzz/index_directory.clj
+++ src/bzzz/index_directory.clj
@@ -11,20 +11,9 @@
   (:require [clojure.java.io :as io])
   (:import (java.io StringReader File)
            (java.lang OutOfMemoryError)
-           (org.apache.lucene.analysis.tokenattributes CharTermAttribute)
-           (org.apache.lucene.facet FacetsConfig FacetField FacetsCollector LabelAndValue)
-           (org.apache.lucene.facet.taxonomy FastTaxonomyFacetCounts)
            (org.apache.lucene.facet.taxonomy.directory DirectoryTaxonomyWriter DirectoryTaxonomyReader)
-           (org.apache.lucene.analysis Analyzer TokenStream)
-           (org.apache.lucene.document Document Field Field$Index Field$Store
-                                       IntField LongField FloatField DoubleField)
-           (org.apache.lucene.search.highlight Highlighter QueryScorer
-                                               SimpleHTMLFormatter TextFragment)
-           (org.apache.lucene.index IndexWriter IndexReader Term IndexableField
-                                    IndexWriterConfig DirectoryReader FieldInfo)
-           (org.apache.lucene.search Query ScoreDoc SearcherManager IndexSearcher
-                                     Explanation Collector TopScoreDocCollector
-                                     TopDocsCollector MultiCollector FieldValueFilter)
+           (org.apache.lucene.index IndexWriter IndexReader IndexWriterConfig DirectoryReader)
+           (org.apache.lucene.search Query ScoreDoc SearcherManager IndexSearcher)
            (org.apache.lucene.store NIOFSDirectory Directory)))

 (declare use-searcher)
@@ -34,7 +23,7 @@

 (def root* (atom default-root))
 (def identifier* (atom default-identifier))
-(def mapping* (atom {}))
+(def name->smanager* (atom {}))

 (def acceptable-name-pattern (re-pattern "[^a-zA-Z_0-9-]"))
 (def shard-suffix "-shard-")
@@ -88,10 +77,10 @@
     (str (as-str index) shard-suffix (str shard))))

 (defn reset-search-managers []
-  (doseq [[name ^SearcherManager manager] @mapping*]
+  (doseq [[name ^SearcherManager manager] @name->smanager*]
     (log/info "\tclosing: " name " " manager)
     (.close manager))
-  (reset! mapping* {}))
+  (reset! name->smanager* {}))

 (defn use-taxo-reader [index callback]
   ;; FIXME: reuse
@@ -108,7 +97,7 @@

 (defn use-searcher [index callback]
   (let [manager (get-search-manager index)
-        searcher (.acquire manager)]
+        searcher ^IndexSearcher (.acquire manager)]
     (try
       (use-taxo-reader index (fn [^DirectoryTaxonomyReader reader] (callback searcher reader)))
       (finally (.release manager searcher)))))
@@ -179,16 +168,16 @@
       (log/warn (ex-str e)))))

 (defn get-search-manager ^SearcherManager [index]
-  (locking mapping*
-    (when (nil? (@mapping* index))
-      (swap! mapping* assoc index (SearcherManager. (new-index-directory (root-identifier-path) index)
+  (locking name->smanager*
+    (when (nil? (@name->smanager* index))
+      (swap! name->smanager* assoc index (SearcherManager. (new-index-directory (root-identifier-path) index)
                                                     nil)))
-    (@mapping* index)))
+    (@name->smanager* index)))

 (defn refresh-search-managers []
   (bootstrap-indexes)
-  (locking mapping*
-    (doseq [[index ^SearcherManager manager] @mapping*]
+  (locking name->smanager*
+    (doseq [[index ^SearcherManager manager] @name->smanager*]
       (log/debug "refreshing: " index " " manager)
       ;; FIXME - reopen cached taxo readers
       (try
@@ -197,16 +186,16 @@
           (do
             (log/info (str index " maybeRefresh exception, closing it. Exception: " (ex-str e)))
             (.close manager)
-            (swap! mapping* dissoc index)))))))
+            (swap! name->smanager* dissoc index)))))))

 (defn shutdown []
-  (locking mapping*
-    (log/info "executing shutdown hook, current mapping: " @mapping*)
+  (locking name->smanager*
+    (log/info "executing shutdown hook, current mapping: " @name->smanager*)
     (reset-search-managers)
-    (log/info "mapping after cleanup: " @mapping*)))
+    (log/info "mapping after cleanup: " @name->smanager*)))

 (defn index-stat []
-  (into {} (for [[name searcher] @mapping*]
+  (into {} (for [[name searcher] @name->smanager*]
              [name (use-searcher name
                                  (fn [^IndexSearcher searcher ^DirectoryTaxonomyReader taxo-reader]
                                    (let [reader (.getIndexReader searcher)]


Adds footer to common/layout.

Adds footer to common/layout.

--- src/cadence/views/common.clj
+++ src/cadence/views/common.clj
@@ -90,7 +90,11 @@
      [:div#main.container
       (when-let [{t :type c :class m :message} (flash/get)]
         (alert (if (nil? c) t c) t m))
-      content]]))
+      content
+      [:footer#footer.footer
+       [:a.label.label-inverse {:href "/#privacy"} "Privacy"]
+       [:a.label {:href "/#security"} "Security"] " "
+       "© 2012 Ryan McGowan"]]]))

 (defn format-errors
   "Takes a collection of error messages and formats it into html."


collect into map with string index

collect into map with string index

--- ohack/src/ohack/core.clj
+++ ohack/src/ohack/core.clj
@@ -277,7 +277,11 @@
     (println "============")
     (doseq [[string-index tab-line] (map-indexed vector line-set)]
       ;; now we need to parse the tab-line...
-      (prn string-index (into {} (parse-guitar-tab-line tab-line)))
+      ;; collect each string's results into into {}
+      (prn string-index (into {}
+                              (map (fn [[k v]] (if v
+                                                [k [string-index v]]))
+                                   (parse-guitar-tab-line tab-line))))
       ))

   )


couple of queries to be used in views later

--- src/clj/apollo/db/schema.clj
+++ src/clj/apollo/db/schema.clj
@@ -142,3 +142,9 @@
   (let [f (io/file path)
         p (if (.isDirectory f) path (.getParent f))]
     (sql/query db [(format "select distinct artist_canonical, album_canonical from tracks where path like '%s%%' order by artist_canonical, album_canonical" p)])))
+
+(defn get-albums-recently-added [db]
+  (sql/query db ["select distinct artist, album from tracks order by last_modified desc,id desc limit 100"]))
+
+(defn get-albums-by-year [db]
+  (sql/query the-db ["select distinct album, artist, year from tracks order by year, artist"]))


improve compact implementation

improve compact implementation

--- src/clj/com/rpl/specter.cljc
+++ src/clj/com/rpl/specter.cljc
@@ -1456,10 +1456,11 @@
                coll? [ALL-WITH-META p]
                )))

-(let [compact* (stay-then-continue (if-path empty? (terminal-val NONE)))]
+(let [empty->NONE (if-path empty? (terminal-val NONE))
+      compact* (fn [nav] (multi-path nav empty->NONE))]
  (defdynamicnav compact
    "During transforms, after each step of navigation in subpath check if the
     value is empty. If so, remove that value by setting it to NONE."
    [& path]
-   (interleave (repeat (count path) compact*) path)
+   (map compact* path)
    ))


Fix cross-process ns creation bug

Fix cross-process ns creation bug

--- src/clj/reply/eval_state.clj
+++ src/clj/reply/eval_state.clj
@@ -21,17 +21,9 @@
 (defn get-ns []
   (@bindings #'*ns*))

-(defn ensure-ns-present [ns]
-  (try
-    (the-ns (symbol ns))
-    (catch Exception e
-      (let [original-ns *ns*]
-        (in-ns ns)
-        (in-ns original-ns)))))
-
 (defmulti set-ns type)
 (defmethod set-ns :default [ns]
-  (ensure-ns-present ns)
+  (create-ns (symbol ns))
   (set-ns (the-ns (symbol ns))))

 (defmethod set-ns clojure.lang.Namespace [ns]


Split scripts

Split scripts

--- src/clj/subman/web/views.clj
+++ src/clj/subman/web/views.clj
@@ -62,5 +62,5 @@
             (when is-debug
               [:script "goog.require('subman.core');"])]
            [:body [:div#main]
-            [:script (str "subman.core.run();"
-                          (get-ga-code ga-id))]])))
+            [:script "subman.core.run();"]
+            [:script (get-ga-code ga-id)]])))


perpetuate the problem->message hack

perpetuate the problem->message hack

--- src/clj_money/validation.clj
+++ src/clj_money/validation.clj
@@ -131,7 +131,9 @@
 (defn- flatten-multi-spec-paths
   [problem]
   (if (and (< 1 (count (:via problem)))
-           (< 1 (count (:path problem))))
+           (< 1 (count (:path problem)))
+           (= (count (:via problem))
+              (count (:path problem)))) ; this is getting very hacky
     (-> problem
         (update-in [:path] rest)
         (update-in [:via] rest))


fix ai, diminish difficulty factor

fix ai, diminish difficulty factor

--- src/cljs/triboard/ai/ai.cljs
+++ src/cljs/triboard/ai/ai.cljs
@@ -33,8 +33,6 @@

 ;; -----------------------------------------

-;; TODO - Try to develop a transducer for min-max
-
 (defn- min-max-step-with
   [min-fn max-fn ai turn on-transition]
   (apply
@@ -46,7 +44,9 @@
 (defn- min-max-step-by
   [key-fn ai turn on-transition]
   (min-max-step-with
-    #(min-key key-fn %) #(max-key key-fn %) ai turn on-transition))
+    (partial min-key key-fn) ;; Lambda does not work here
+    (partial max-key key-fn)
+    ai turn on-transition))

 (defn- min-max-step
   [ai turn on-transition]
@@ -93,7 +93,7 @@

 (defn- focus-human-player?
   [{:keys [blue red green] :as scores}]
-  (let [human-score (* 1.2 blue)]
+  (let [human-score (* 1.1 blue)]
     (and (< red human-score) (< green human-score))))

 (defn play-best-move


Kibitzed.

Kibitzed.

--- src/com/lemonodor/gflags.clj
+++ src/com/lemonodor/gflags.clj
@@ -252,7 +252,7 @@
   (loop [parsed-file-list []
          rest-of-args args
          new-argv []]
-    (if (not (seq rest-of-args))
+    (if-not (seq rest-of-args)
       new-argv
       (let [current-arg (first rest-of-args)
             rest-of-args (rest rest-of-args)]
@@ -262,7 +262,7 @@
                         (= current-arg "-flagfile"))
                   ;; This handles the case of -(-)flagfile foo.  In this
                   ;; case the next arg really is part of this one.
-                  (if (not (seq rest-of-args))
+                  (if-not (seq rest-of-args)
                     (throw (Exception. "--flagfile with no argument"))
                     [(fs/expand-home (first rest-of-args)) (rest rest-of-args)])
                   ;; This handles the case of (-)-flagfile=foo


Add helper for find dom objects by class name.

--- frontend/uxbox/ui/dom.cljs
+++ frontend/uxbox/ui/dom.cljs
@@ -1,4 +1,9 @@
-(ns uxbox.ui.dom)
+(ns uxbox.ui.dom
+  (:require [goog.dom :as dom]))
+
+(defn get-element-by-class
+  [^string classname]
+  (dom/getElementByClass classname))

 (defn stop-propagation
   [e]


fixup! selected-tab-name is a clearer name

--- src-cljs/frontend/components/pieces/tabs.cljs
+++ src-cljs/frontend/components/pieces/tabs.cljs
@@ -8,11 +8,12 @@
   "A row of tabs, suitable for the top of a card.

   :tabs              - A sequence of tabs, in display order. Each tab is a map:
-                      :name  - A unique identifier for the tab, not displayed to the user.
-                      :icon  - An icon which appears next to the tab label, often an <i>
-                               element of some sort.
-                      :label - The text which labels the tab. This may also be a component
-                               or a list of components.
+                       :name  - A unique identifier for the tab, not displayed to the
+                                user.
+                       :icon  - An icon which appears next to the tab label, often an <i>
+                                element of some sort.
+                       :label - The text which labels the tab. This may also be a
+                                component or a list of components.
   :selected-tab-name - The name of the selected tab.
   :on-tab-click      - A handler called when a tab is clicked. The handler will receive
                        the name of the clicked tab."


Updating to include the standard templates and content to start with a reference

--- src/conceptnet_clj/views.clj
+++ src/conceptnet_clj/views.clj
@@ -17,3 +17,90 @@

 (defn render [& args]
   (apply str (flatten (map emit args))))
+
+(deftemplate main-template "templates/index.html"
+  [& {:keys [body title]}]
+  [:head :title] (content (str/join " - "
+                                    (filter identity [title "ConceptNet"])))
+  [:#container :> #{:h1 :h2} :a] (set-attr :href "/")
+  [:#container :ul] (substitute body))
+
+(defsnippet form "templates/search_form.html" [:#container :form]
+  [person1 person2 & args]
+  [:form] (do->
+            (set-attr :action "search")
+            (set-attr :method "GET"))
+  (form-input 1) (do->
+                   (set-attr :name "person1")
+                   (set-attr :value person1))
+  (form-input 2) (do->
+                   (set-attr :name "person2")
+                   (set-attr :value person2)))
+
+(defsnippet possibility "templates/disambiguate.html" [:#disambiguate [:li (nth-of-type 1)]]
+  [[person1 person2]]
+  [:a] (set-attr :href (str (url "/search" {:person1 (:name person1) :person2 (:name person2)})))
+  [:a :> html/text-node] (html/wrap :rel)
+  [:a :> [:rel html/first-child]] (content (format-name (:name person1)))
+  [:a :> [:rel html/last-child]] (content (format-name (:name person2)))
+  [:a :> :rel] html/unwrap)
+
+(defsnippet possibilities "templates/disambiguate.html" [:#disambiguate]
+  [pairs]
+  [:ul] (content (map possibility pairs)))
+
+(def ^:dynamic *result-sel* [:.result_list :> [:ul (nth-of-type 1)] :> [:li (nth-of-type 1)]])
+
+(defn simple-escape [^String text]
+  (.. text (replace " " "+") (replace "&" "%26")))
+
+(defn- imdb-link [{:keys [name type]}]
+  (str "http://imdb.com/find?exact="
+       (= type "movie")
+       "&q="
+       (simple-escape name)))
+
+(defn- display-name [{:keys [name type]}]
+  (if (= type "actor")
+    (format-name name)
+    name))
+
+(defsnippet result-node "templates/results.html"  (conj *result-sel* [:li (nth-of-type 1)])
+  [node]
+  [:li] (set-attr :class (:type node))
+  [:li :a] (do->
+             (set-attr :href (imdb-link node))
+             (content (display-name node))))
+
+(defsnippet result "templates/results.html" *result-sel*
+  [path]
+  [:ul] (content (map result-node path)))
+
+(defn degrees-description [bacon-number]
+  (let [degrees (str bacon-number " degree" (when-not (= 1 bacon-number) "s"))]
+    (str "(with " degrees " of separation)")))
+
+(defsnippet results "templates/results.html" [:#results]
+  [{:keys [paths total start end bacon-number]}]
+  [:.result_list :> [:ul (html/but first-of-type)]] nil
+  [:.result_list :> [:ul first-of-type]] (clone-for [path paths]
+                                                    (content (result path)))
+  [:.result_list :> :h3] (content (str (count paths) " path" (when-not (= 1 (count paths)) "s")))
+  [:.result_list :> :h4] (content (degrees-description bacon-number))
+  [:.bacon_number :mark] (content (str bacon-number))
+  [:.bacon_number [:p first-of-type]] (content (format-name start))
+  [:.bacon_number [:p last-of-type]] (content (format-name end)))
+
+(defn disambiguate [pairs {:keys [hard-mode person1 person2]}]
+  (main-template :title "Did you mean one of these?"
+                 :body [(if (seq pairs)
+                          (possibilities pairs)
+                          (html/html [:h2 "Oops. We couldn't find one of those actors"]))
+                        (html/html [:h2 "Try a new search"])
+                        (form person1 person2 hard-mode)]))
+
+(defn results-page [{:keys [paths start end] :as res}]
+  (main-template :body (if (seq paths)
+                         (results res)
+                         "Not linkable in 5 hops or fewer")
+                 :title (str "From " start " to " end)))


update dep versions

update dep versions

--- kafka-clj/project.clj
+++ kafka-clj/project.clj
@@ -15,7 +15,7 @@
   :jvm-opts ["-Xmx3g"]
   :plugins [
          [lein-rpm "0.0.5"] [lein-midje "3.0.1"] [lein-marginalia "0.7.1"]
-	 [lein-cloverage "1.0.2"]
+	       [lein-cloverage "1.0.2"]
          [lein-kibit "0.0.8"] [no-man-is-an-island/lein-eclipse "2.0.0"]
            ]
   :dependencies [


Whitespace

Whitespace

--- src/day8/re_frame/trace.cljs
+++ src/day8/re_frame/trace.cljs
@@ -40,7 +40,7 @@
                                               (ratom/run-in-reaction #(reagent.impl.component/do-render c) c "cljsRatom"
                                                                      batch/queue-render reagent.impl.component/rat-opts)
                                               (._run rat false))
-                                 cljs-ratom ($ c :cljsRatom)]               ;; actually a reaction
+                                 cljs-ratom ($ c :cljsRatom)] ;; actually a reaction
                              (trace/merge-trace!
                                {:tags {:reaction      (rutils/reagent-id cljs-ratom)
                                        :input-signals (when cljs-ratom
@@ -73,11 +73,11 @@
             (case key
               :componentWillUnmount
               (fn [] (this-as c
-                              (trace/with-trace {:op-type   key
-                                                 :operation (last (str/split (comp-name c) #" > "))
-                                                 :tags      {:component-path (reagent.impl.component/component-path c)
-                                                             :reaction       (rutils/reagent-id ($ c :cljsRatom))}})
-                              (.call (real-custom-wrapper key f) c c)))
+                       (trace/with-trace {:op-type   key
+                                          :operation (last (str/split (comp-name c) #" > "))
+                                          :tags      {:component-path (reagent.impl.component/component-path c)
+                                                      :reaction       (rutils/reagent-id ($ c :cljsRatom))}})
+                       (.call (real-custom-wrapper key f) c c)))

               (real-custom-wrapper key f))))

@@ -100,13 +100,13 @@
       (not (str/includes? (or (get-in trace [:tags :component-path]) "") "day8.re_frame.trace")))

-   #_ (if-let [comp-p (get-in trace [:tags :component-path])]
-      (println comp-p))))
+    #_(if-let [comp-p (get-in trace [:tags :component-path])]
+        (println comp-p))))

 (defn init-tracing! []
   (re-frame.trace/register-trace-cb ::cb (fn [new-traces]
-                                             (let [new-traces (filter log-trace? new-traces)]
-                                               (swap! traces #(reduce conj % new-traces)))))
+                                           (let [new-traces (filter log-trace? new-traces)]
+                                             (swap! traces #(reduce conj % new-traces)))))
   (monkey-patch-reagent)
   )

@@ -160,7 +160,7 @@
            (doall
              (for [{:keys [op-type id operation tags duration] :as trace} showing-traces]
                (let [row-style (merge padding {:border-top (case op-type :event "1px solid lightgrey" nil)})
-                     #_ #_ _         (js/console.log (devtools/header-api-call tags))
+                     #_#__ (js/console.log (devtools/header-api-call tags))
                      ]
                  (list [:tr {:key   id
                              :style {:color (case op-type
@@ -211,7 +211,7 @@
       {:component-will-mount   #(js/window.addEventListener "keydown" handle-keys)
        :component-will-unmount #(js/window.removeEventListener "keydown" handle-keys)
        :display-name           "devtools outer"
-       :reagent-render       (fn []
+       :reagent-render         (fn []
                                  (let [draggable-area 10
                                        full-width     js/window.innerWidth
                                        full-height    js/window.innerHeight


added console output as side effect for generating TODO placeholders in HTML content

--- src/distillery/view/html.clj
+++ src/distillery/view/html.clj
@@ -122,4 +122,7 @@
               :attrs {:class "innerpage"}
               :content (safe-content content)}]})

-(defn TODO [text] {:tag :div :attrs {:class "todo"} :content text})+(defn TODO
+  [text]
+  (println (format "TODO: %s" text))
+  {:tag :div :attrs {:class "todo"} :content text})


chin-up challenge requires increasing the batch size

--- src/ontrail/group.clj
+++ src/ontrail/group.clj
@@ -133,7 +133,8 @@
     (mq/with-collection EXERCISE
        (mq/find query)
        (mq/fields [:creationDate :detailRepeats :user])
-       (mq/sort {:creationDate 1}))))
+       (mq/sort {:creationDate 1})
+       (mq/batch-size 2000))))

 (defn analyze-chinups [day-number all-results user]
   (let [results (filter #(= user (:user %)) all-results)


relax ID type constraints for cross validation

--- src/clojure/zensols/model/eval_classifier.clj
+++ src/clojure/zensols/model/eval_classifier.clj
@@ -557,7 +557,7 @@
        (map (fn [row]
               (-> insts
                   (.instance row)
-                  (.stringValue id-attrib) Integer/parseInt)))))
+                  (.stringValue id-attrib))))))

 (defn two-pass-train-instances
   "Don't use this function--instead, use [[with-two-pass]].


Add the ability to send a response body in multiple chunks

--- src/picard/server.clj
+++ src/picard/server.clj
@@ -9,6 +9,7 @@
    [org.jboss.netty.channel
     Channel]
    [org.jboss.netty.handler.codec.http
+    DefaultHttpChunk
     DefaultHttpResponse
     HttpChunk
     HttpHeaders
@@ -41,6 +42,10 @@
       (.setContent resp (formats/string->channel-buffer body)))
     resp))

+(defn- body-to-netty-chunk
+  [body]
+  (DefaultHttpChunk. (formats/string->channel-buffer body)))
+
 (defn- body-from-netty-req
   [^HttpMessage req]
   (.getContent req))
@@ -52,16 +57,24 @@

 (defn- stream-or-finalize-resp
   [evt val req-state chan keepalive? streaming? last-write]
-  (when (= :body evt)
-    (throw (Exception. "Not implemented yet...")))
+  (cond
+   (= :body evt)
+   (let [write (.write chan (body-to-netty-chunk val))]
+     #(stream-or-finalize-resp %1 %2
+                               req-state chan
+                               keepalive? true write))

-  (if keepalive?
-    (swap! req-state (fn [[current args]]
-                       (if (= current waiting-for-response)
-                         [incoming-request args]
-                         [current args true])))
-    (.addListener last-write netty/close-channel-future-listener))
-  (fn [& args] (throw (Exception. "This request is finished"))))
+   (= :done evt)
+   (do (if keepalive?
+         (swap! req-state (fn [[current args]]
+                            (if (= current waiting-for-response)
+                              [incoming-request args]
+                              [current args true])))
+         (.addListener last-write netty/close-channel-future-listener))
+       (fn [& args] (throw (Exception. "This request is finished"))))
+
+   :else
+   (throw (Exception. "Unknown event: " evt))))

 (defn- is-keepalive?
   [req-keepalive? hdrs]


Fix typo

Fix typo

--- src/cats/context.cljc
+++ src/cats/context.cljc
@@ -36,7 +36,7 @@
      :clj  (throw (IllegalArgumentException. text))))

 (defn context?
-  "Returnt `true` if the provided value satisfies
+  "Return `true` if the provided value satisfies
   the Context protocol."
   [v]
   (satisfies? p/Context v))


changing to actions

changing to actions

--- src/celestial/api.clj
+++ src/celestial/api.clj
@@ -101,7 +101,7 @@

 (defroutes app-routes
-  hosts types tasks jobs (friend/wrap-authorize users admin) (friend/wrap-authorize quotas admin) (route/not-found "Not Found"))
+  hosts types actions jobs (friend/wrap-authorize users admin) (friend/wrap-authorize quotas admin) (route/not-found "Not Found"))

 (defn error-wrap
   "A catch all error handler"


CSVs now stream successfully in the browser

--- src/cfpb/qu/views.clj
+++ src/cfpb/qu/views.clj
@@ -438,7 +438,7 @@
                      (response/header "X-Computing" computing))]
     (if (query/valid? query)
       (if (should-stream? resource)
-        (stream-slice-query-csv request (respond "") (concat (vector columns) rows))
+        (stream-slice-query-csv request (respond (write-csv (vector columns))) rows)
         (respond (str (write-csv (vector columns)) (write-csv rows))))
       (response/content-type (response/response "") "text/plain"))))


made chip settings explicit and self-documenting

--- src/hsec/source/mcp23017.clj
+++ src/hsec/source/mcp23017.clj
@@ -1,14 +1,17 @@
 ;; All things specific to the mcp23017 chip. If you have to read the datasheet
 ;; or access the chip, it goes in this file. All else, no!
 ;; Datasheet: https://www.microchip.com/wwwproducts/en/MCP23017
+;; The Microchip MCP23017 has 16-bit I/O port functionality via
+;; two 8-bit ports (PORTA and PORTB). This should not be confused
+;; with addressing modes BANK=0 or BANK=1.

 (ns hsec.source.mcp23017
   (:require [dvlopt.linux.i2c       :as i2c]
             [dvlopt.linux.i2c.smbus :as smbus]))

 (def registers
-  {
-   :a  { ;; addressing mode 'a'
+  {  ;; BANK=0 addressing (registers paired)
+   :a  { ;; 'a' register, first 8 IO bits
         :iodir 0x00
         :ipol 0x02
         :gpinten 0x04
@@ -21,7 +24,7 @@
         :gpio 0x12
         :olat 0x14
         }
-   :b { ;; addressing mode 'b'
+   :b { ;; 'b' register, second 8 IO bits
        :iodir 0x01
        :ipol 0x03
        :gpinten 0x05
@@ -35,6 +38,41 @@
        :olat 0x15
        }
    })
+
+(def iocon-settings
+  [
+   0 ;; BANK: Controls how the registers are addressed
+   ;; 1 = The registers associated with each port are separated into different
+   ;;     banks.
+   ;; 0 = The registers are in the same bank (addresses are sequential).
+
+   1 ;; MIRROR: INT Pins Mirror bit
+   ;; 1 = The INT pins are internally connected
+   ;; 0 = The INT pins are not connected. INTA is associated with PORTA and INTB
+   ;;     is associated with PORTB
+
+   0 ;; SEQOP: Sequential Operation mode bit
+   ;; 1 = Sequential operation disabled, address pointer does not increment.
+   ;; 0 = Sequential operation enabled, address pointer increments.
+
+   0 ;; DISSLW: Slew Rate control bit for SDA output
+   ;; 1 = Slew rate disabled
+   ;; 0 = Slew rate enabled
+
+   0 ;; HAEN: Hardware Address Enable bit (MCP23S17 only) (Note 1)
+   ;; 1 = Enables the MCP23S17 address pins.
+   ;; 0 = Disables the MCP23S17 address pins.
+
+   0 ;; ODR: Configures the INT pin as an open-drain output
+   ;; 1 = Open-drain output (overrides the INTPOL bit.)
+   ;; 0 = Active driver output (INTPOL bit sets the polarity.)
+
+   1 ;; INTPOL: This bit sets the polarity of the INT output pin
+   ;; 1 = Active-high
+   ;; 0 = Active-low
+
+   0 ;; Unimplemented: Read as ‘0’
+])

 (defn get-register
   "return the byte contents of a register"
@@ -81,13 +119,12 @@
     ;; this bus will always use the selected slave address for commands
     (i2c/select-slave bus slave-address)

-    ;; set int mirror and active-high
-    ;; The 4 bit causes the interrupt pins to be internally connected so
-    ;; that either interrupt causes both to be on.
-    ;; The active-high causes an interrupt to have high polarity when active.
-    ;; The IOCON register is shared between :a and :b so it only needs to be
-    ;; set for one bank.
-    (smbus/write-byte bus (get-in registers [:a :iocon]) 0x42)
+    ;; setup the chip with settings defined above in the iocon-settings object
+    ;; in particular, the interrupt pins should be mirrored and driven high
+    ;; when there's an interrupt.
+    (smbus/write-byte bus
+                      (get-in registers [:a :iocon])
+                      (Integer/parseInt (apply str iocon-settings) 2))

     ;; enable int on all pins
     ;; (smbus/write-byte bus (get-in mcp23017/register [:a :gpinten]) 0x04)


Remove debugging

Remove debugging

--- src/clj/runbld/io.clj
+++ src/clj/runbld/io.clj
@@ -58,7 +58,6 @@
                     (System/gc)
                     (.setWritable x true))
                   (again/with-retries [100 500 500]
-                    (println "deleting" x)
                     (jio/delete-file x))
                   (recur xs))


fix defn arg lists, fix env

fix defn arg lists, fix env

--- src/semitone/core.clj
+++ src/semitone/core.clj
@@ -9,7 +9,6 @@
 ;;    - get a unique set of channels from the intersecting notes
 ;;    - choose to play on the next available channel on the channel list
 ;;    - if no channels are available, play on the next channel/least used channel
-;;   fix envelopes
 ;;   :time-signature [4 4]
 ;;   add shortcuts for intervals, chords, key signatures, dynamics
 ;;   DAW recording interface for capturing param values / performances


deleted commented out code

deleted commented out code

--- src/server/rooms.cljs
+++ src/server/rooms.cljs
@@ -366,15 +366,6 @@
         (if (empty? info-to-send)
           (callback nil)
           (callback (clj->js info-to-send)))))))
-(comment
-  (.hgetall redis-client (redis-room-prefix room-id "music-info")
-    (fn [err music-info-reply]
-      (let [info (vals (js->clj music-info-reply))
-            reader (transit/reader :json)
-            info-to-send (map #(transit/read reader %1) info)]
-        (if (empty? info-to-send)
-          (callback nil)
-          (callback  (clj->js info-to-send)))))))

 (defn track-complete [room callback]
   (.set redis-client (redis-room-prefix room "started?") "false"


Added storage changes cmd handler placeholder.

--- src/cbfg/world-storage.cljs
+++ src/cbfg/world-storage.cljs
@@ -7,37 +7,37 @@
             [cbfg.fence :refer [make-fenced-pump]]))

 (defn make-storage-cmd-handlers [kvs changes last-seq max-deleted-seq]
-  {"get"  [["key"]
-           (fn [c] {:rq
-                    (fn [actx]
-                      (ago storage-get actx
-                           {:opaque-id (:opaque-id c) :status :ok
-                            :key (:key c) :val "yay"}))})]
-   "set"  [["key" "val"]
-           (fn [c] {:rq
-                    (fn [actx]
-                      (ago storage-get actx
-                           {:opaque-id (:opaque-id c) :status :ok
-                            :key (:key c)}))})]
-   "del"  [["key"]
-           (fn [c] {:rq
-                    (fn [actx]
-                      (ago storage-get actx
-                           {:opaque-id (:opaque-id c) :status :ok
-                            :key (:key c)}))})]
+  {"get" [["key"]
+          (fn [c] {:rq (fn [actx]
+                         (ago storage-get actx
+                              {:opaque-id (:opaque-id c) :status :ok
+                               :key (:key c) :val "yay"}))})]
+   "set" [["key" "val"]
+          (fn [c] {:rq (fn [actx]
+                         (ago storage-get actx
+                              {:opaque-id (:opaque-id c) :status :ok
+                               :key (:key c)}))})]
+   "del" [["key"]
+          (fn [c] {:rq (fn [actx]
+                         (ago storage-get actx
+                              {:opaque-id (:opaque-id c) :status :ok
+                               :key (:key c)}))})]
    "scan" [["from" "to"]
-           (fn [c] {:rq
-                    (fn [actx]
-                      (ago storage-get actx
-                           {:opaque-id (:opaque-id c) :status :ok
-                            :from (:from c) :to (:to c)}))})]
-   "noop" [[]
-           (fn [c] {:rq
-                    (fn [actx]
-                      (ago storage-get actx
-                           {:opaque-id (:opaque-id c) :status :ok}))})]
-   "test" [[]
-           (fn [c] {:rq #(cbfg.fence/test % (:opaque-id c))})]})
+           (fn [c] {:rq (fn [actx]
+                          (ago storage-get actx
+                               {:opaque-id (:opaque-id c) :status :ok
+                                :from (:from c) :to (:to c)}))})]
+   "changes" [["since"]
+              (fn [c] {:rq
+                       (fn [actx]
+                         (ago storage-get actx
+                              {:opaque-id (:opaque-id c) :status :ok
+                               :since (:since c)}))})]
+   "noop" [[] (fn [c] {:rq
+                       (fn [actx]
+                         (ago storage-get actx
+                              {:opaque-id (:opaque-id c) :status :ok}))})]
+   "test" [[] (fn [c] {:rq #(cbfg.fence/test % (:opaque-id c))})]})

 (def storage-max-inflight (atom 10))


Tracking max-deleted-sq in storage-del, using in storage-set.

--- src/cbfg/world-storage.cljs
+++ src/cbfg/world-storage.cljs
@@ -13,7 +13,7 @@
   (let [s @storage
         sq (get (:keys s) key)
         change (get (:changes s) sq)]
-    (if (and change (not (:deletion change)))
+    (if (and change (not (:deleted change)))
       {:opaque opaque :status :ok
        :key key :sq sq :cas (:cas change) :val (:val change)}
       {:opaque opaque :status :not-found
@@ -23,7 +23,9 @@
   (let [cas (gen-cas)
         storage2 (swap! storage
                         #(-> %
-                             (dissoc-in [:changes (get-in % [:keys key :sq])])
+                             (assoc :next-sq (max (:next-sq %)
+                                                  (inc (:max-deleted-sq %))))
+                             (dissoc-in [:changes (get-in % [:keys key])])
                              (assoc-in [:keys key] (:next-sq %))
                              (assoc-in [:changes (:next-sq %)]
                                        {:key key :sq (:next-sq %) :cas cas :val val})
@@ -34,23 +36,15 @@
 (defn storage-del [storage opaque key]
   (let [storage2 (swap! storage
                         #(-> %
-                             (dissoc-in [:changes (get-in % [:keys key :sq])])
+                             (assoc :max-deleted-sq (max (get-in % [:keys key])
+                                                         (:max-deleted-sq %)))
+                             (dissoc-in [:changes (get-in % [:keys key])])
                              (dissoc-in [:keys key])
                              (assoc-in [:changes (:next-sq %)]
-                                       {:key key :sq (:next-sq %) :deletion true})
+                                       {:key key :sq (:next-sq %) :deleted true})
                              (update-in [:next-sq] inc)))]
     {:opaque opaque :status :ok
      :key key :sq (dec (:next-sq storage2))}))
-
-(defn storage-scan [storage opaque from to]
-  (let [s @storage
-        sq (get (:keys s) key)
-        change (get (:changes s) sq)]
-    (if (and change (not (:deletion change)))
-      {:opaque opaque :status :ok
-       :key key :sq sq :cas (:cas change) :val (:val change)}
-      {:opaque opaque :status :not-found
-       :key key})))

 (defn make-storage-cmd-handlers [storage]
   {"get" [["key"]
@@ -90,8 +84,11 @@
                          (get-el-innerHTML (str el-prefix "-" log-kind "-log")))))

 (defn world-vis-init [el-prefix]
-  (let [storage (atom {:keys (sorted-map) :changes (sorted-map)
-                       :next-sq 1 :max-deleted-sq 1})
+  (let [storage (atom {:keys (sorted-map)    ; key -> sq
+                       :changes (sorted-map) ; sq -> {:key k, :sq s,
+                                             ;        <:cas c, :val v> | :deleted bool}
+                       :next-sq 1
+                       :max-deleted-sq 1})
         storage-cmd-handlers (make-storage-cmd-handlers storage)
         cmd-ch (map< (fn [ev] {:op (.-id (.-target ev))})
                      (merge (map #(listen-el (gdom/getElement %) "click")


Make compatible dsl functions with custom user defined types.

--- src/suricatta/dsl.clj
+++ src/suricatta/dsl.clj
@@ -26,6 +26,7 @@
   "Sql building dsl"
   (:refer-clojure :exclude [val group-by and or not name set])
   (:require [suricatta.core :as core]
+            [suricatta.impl :as impl]
             [suricatta.types :as types :refer [defer]]
             [suricatta.proto :as proto])
   (:import org.jooq.impl.DSL
@@ -192,10 +193,12 @@

 (extend-protocol IDeferred
   Object
-  (unwrap* [self] self)
+  (unwrap* [self]
+    (impl/wrap-if-need self))

   suricatta.types.Deferred
-  (unwrap* [self] @self))
+  (unwrap* [self]
+    (unwrap* @self)))

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Common DSL functions
@@ -460,7 +463,8 @@
 (defn insert-values
   [t values]
   (defer
-    (-> (fn [acc [k v]] (.set acc (field* k) (unwrap* v)))
+    (-> (fn [acc [k v]]
+          (.set acc (field* k) (unwrap* v)))
         (reduce (unwrap* t) values)
         (.newRecord))))


formatting, use loop instead of immediately invoked fn

--- src/chat/client/router.cljs
+++ src/chat/client/router.cljs
@@ -16,18 +16,19 @@
     (defn go-to [path]
       (. h (setToken path)))

-    (events/listen js/document.body "click" (fn [e]
-                                              ; if this element has href, or any parent has href, then pushstate
-                                              (when-let [href ((fn [e]
-                                                                 (if-let [href (.-href e)]
-                                                                   href
-                                                                   (when-let [parent (.-parentNode e)]
-                                                                     (recur parent)))) (.-target e))]
-                                                (when (.hasSameDomainAs (.parse Uri href) (.parse Uri js/window.location))
-                                                  (let [path (.getPath (.parse Uri href))]
-                                                    (when (secretary/locate-route path)
-                                                      (. h (setToken path))
-                                                      (.preventDefault e)))))))))
+    (events/listen js/document.body "click"
+      (fn [e]
+        ; if this element has href, or any parent has href, then pushstate
+        (when-let [href (loop [e (.-target e)]
+                          (if-let [href (.-href e)]
+                            href
+                            (when-let [parent (.-parentNode e)]
+                              (recur parent))))]
+          (when (.hasSameDomainAs (.parse Uri href) (.parse Uri js/window.location))
+            (let [path (.getPath (.parse Uri href))]
+              (when (secretary/locate-route path)
+                (. h (setToken path))
+                (.preventDefault e)))))))))

 (defn dispatch-current-path! []
   (secretary/dispatch! (.getPath (.parse Uri js/window.location))))


Abort if peer has been unaccepted via leave-cluster

--- src/onyx/log/commands/accept_join_cluster.clj
+++ src/onyx/log/commands/accept_join_cluster.clj
@@ -12,9 +12,10 @@
   (let [{:keys [accepted-joiner accepted-observer]} args
         target (or (get-in replica [:pairs accepted-observer])
                    accepted-observer)
+        accepted? (get-in replica [:accepted accepted-observer])
         already-joined? (some #{accepted-joiner} (:peers replica))
         no-observer? (not (some #{target} (:peers replica)))]
-    (if (or already-joined? no-observer?)
+    (if (or already-joined? no-observer? (not accepted?))
       replica
       (-> replica
           (update-in [:pairs] merge {accepted-observer accepted-joiner})


Add rounding to RuuviTag observation query processing

--- src/env_logger/db.clj
+++ src/env_logger/db.clj
@@ -12,7 +12,8 @@
             [honeysql
              [core :as sql]
              [helpers :refer :all]])
-  (:import java.time.ZoneId
+  (:import java.text.NumberFormat
+           java.time.ZoneId
            org.postgresql.util.PSQLException))

 (let [db-host (get (System/getenv)
@@ -61,7 +62,7 @@
           (f/parse (f/formatter "y-M-d H:mZ")
                    (s/replace (nth match 1) "T" " "))
           (t/now)))
-       diff-minutes)))
+        diff-minutes)))

 (defn get-yc-image
   "Returns the name of the latest yardcam image."
@@ -378,19 +379,22 @@
   and having the given location."
   [db-con start end location]
   (try
-    (let [query (sql/format (sql/build :select [:temperature
+    (let [nf (NumberFormat/getInstance)
+          query (sql/format (sql/build :select [:temperature
                                                 :humidity]
                                        :from :ruuvitag_observations
                                        :where [:and
                                                [:= :location location]
                                                [:>= :recorded start]
                                                [:<= :recorded end]]
-                                       :order-by [[:id :asc]]))
-          result (j/query db-con query
-                          {:row-fn (fn [obs]
-                                     {:rt-temperature (:temperature obs)
-                                      :rt-humidity (:humidity obs)})})]
-      result)
+                                       :order-by [[:id :asc]]))]
+      (.applyPattern nf "0.0#")
+      (j/query db-con query
+               {:row-fn (fn [obs]
+                          {:rt-temperature (Float/parseFloat
+                                            (. nf format (:temperature obs)))
+                           :rt-humidity (Float/parseFloat
+                                         (. nf format (:humidity obs)))})}))
     (catch PSQLException pe
       (log/error "RuuviTag observation fetching failed:"
                  (.getMessage pe))


Define some routes for / and /set-angle

Define some routes for / and /set-angle

--- src/clj/calc_angle/core.clj
+++ src/clj/calc_angle/core.clj
@@ -1,6 +1,14 @@
 (ns calc-angle.core
   (:require [clojure.string :as string]
-            [clojure.core.reducers :as r])
+            [clojure.core.reducers :as r]
+            [ring.middleware.defaults :refer [wrap-defaults site-defaults]]
+            [ring.middleware.params :refer [wrap-params]]
+            [ring.util.response :refer [redirect content-type response]]
+            [compojure.core :refer [GET defroutes]]
+            [compojure.handler :refer [site]]
+            [compojure.route :refer [resources not-found]]
+            [clojure.java.io :as io])
+  (:import  [jssc SerialPort SerialPortList])
   (:gen-class))

 (declare radian-to-degree)
@@ -195,7 +203,7 @@
         y (+ (* (Math/sin beta-rad) first-arm) (* (Math/sin (- alpha-rad)) second-arm))]
     (list (* r (Math/cos theta-rad)) y (* r (Math/sin (- theta-rad))))))

-(defn -main
+(defn -main2
   "main function"
   [& args]
   (with-open [fin (clojure.java.io/reader "resources/huan_strokes.txt")]
@@ -263,3 +271,41 @@
                     (println (str "const int16_t betas[] PROGMEM = {" (string/join ", " (map #(str (second %)) alpha-beta-degree-list)) "};"))
                     (println (str "const int16_t delta_thetas[] PROGMEM = {" (string/join ", " (map #(str (get-base-angle (second %))) r-theta-list)) "};"))))
           (.write fout "\n"))))))
+
+(def ^{:dynamic true} *serial-conn*)
+
+;(defn set-port [idx val]
+; (.writeBytes serial-conn (byte-array [(byte \s) (byte idx) (byte val)]))
+; (aget (.readBytes serial-conn 1) 0))
+;
+;(defn query-port [idx]
+; (.writeBytes serial-conn (byte-array [(byte \g) (byte idx)]))
+; (aget (.readBytes serial-conn 1) 0))
+
+(defroutes app
+           (GET "/" [] (redirect "index.html"))
+           (GET "/set-angle" [req] (if (:query-params req)
+                                     (set-angle req)
+                                     "/set-angle?a=<base>&b=<large>&c="))
+           (resources "/")
+           (not-found "
not found
"))
+
+(def entry (site app))
+
+(defn init
+  "one time initialization work"
+  []
+  (let [ports (. SerialPortList getPortNames)
+        myport "/dev/tty.usbmodem1411"]
+    (if (and (pos? (alength ports)) (contains? (set ports) myport))
+      (set! *serial-conn*
+            (doto (SerialPort. myport)
+              (.openPort)
+              (.setParams 9600 8 1 0)
+              ((fn [_] (Thread/sleep 1000)))))
+      (println "There is no serial connection!"))))
+
+(defn destroy
+  "release acquired resources before service stops"
+  []
+  (.closePort *serial-conn*))


code cleanup

code cleanup

--- src/clj/ceres/collector.clj
+++ src/clj/ceres/collector.clj
@@ -23,7 +23,9 @@
 (def news-accounts #{"FAZ_NET" "dpa" "tagesschau" "SPIEGELONLINE" "SZ" "BILD" "DerWesten" "ntvde" "tazgezwitscher" "welt" "ZDFheute" "N24_de" "sternde" "focusonline"} )

-(defn init-mongo []
+(defn init-mongo
+  "Define mongodb indices on first start"
+  []
   (do
     (mc/ensure-index db "articles" (array-map :ts 1))
     (mc/ensure-index db "origins" (array-map :ts 1))
@@ -54,17 +56,9 @@
                                     :content-type content-type})))))))

-(defn store-url [{:keys [article record ts source]}]
-  (mc/insert-and-return
-   db
-   "urls"
-   {:tweet record
-    :article article
-    :source source
-    :ts ts}))
-
-
-(defn store-origin [{:keys [article record ts source ancestors root]}]
+(defn store-origin
+  "Store relationship between article and tweet"
+  [{:keys [article record ts source]}]
   (mc/insert-and-return
    db
    "origins"
@@ -74,7 +68,9 @@
     :ts ts}))

-(defn store-article [{:keys [url content-type ts] :as expanded-url}]
+(defn store-article
+  "Fetch html document, extract title and store them"
+  [{:keys [url content-type ts] :as expanded-url}]
   (let [raw-html (slurp url)
         html-title (-> (java.io.StringReader. raw-html) enlive/html-resource (enlive/select [:head :title]) first :content first)]
     (mc/insert-and-return
@@ -88,7 +84,7 @@

 (defn store
-  "Stores the given tweet in mongodb"
+  "Store the tweet"
   [tweet]
   (let [oid (ObjectId.)
         doc (update-in tweet [:created_at] (fn [x] (f/parse custom-formatter x)))


now with zippers

now with zippers

--- src/clj/ceres/collector.clj
+++ src/clj/ceres/collector.clj
@@ -3,6 +3,8 @@
   (:require [monger.core :as mg]
             [monger.collection :as mc]
             [monger.operators :refer :all]
+            [clojure.walk :as walk]
+            [clojure.zip :as zip]
             [monger.conversion :refer [from-db-object]]
             [monger.query :refer :all]
             [monger.joda-time]
@@ -456,34 +458,101 @@
            (pmap transact-published)))))

+  ;; retweet reactions
   (time
    (do
+     (say/say "Computation started")
+     (doall
+      (pmap (fn [{:keys [tweet _id]}]
+             (let [{{:keys [id_str]} :retweeted_status} (mc/find-map-by-id @db "tweets" tweet)
+                   {stid :_id} (mc/find-one-as-map @db "tweets" {:id_str id_str})
+                   {spid :_id} (if stid (mc/find-one-as-map @db "published" {:tweet stid})
+                                   nil)]
+               (store-reaction _id spid)))
+           (mc/find-maps @db "published" {:type :retweet})))
+     (say/say "Computation completed!")))
+
+
+  ;; reply reactions
+  (time
+   (do
+     (say/say "Computation started")
+     (doall
+      (pmap (fn [{:keys [tweet _id]}]
+             (let [{:keys [in_reply_to_status_id_str]} (mc/find-map-by-id @db "tweets" tweet)
+                   {stid :_id} (mc/find-one-as-map @db "tweets" {:id_str in_reply_to_status_id_str})
+                   {spid :_id} (if stid (mc/find-one-as-map @db "published" {:tweet stid})
+                                   nil)]
+               (store-reaction _id spid)))
+           (mc/find-maps @db "published" {:type :reply})))
+     (say/say "Computation completed")))
+
+
+  ;; share reactions
+  (time
+   (do
+     (say/say "Computation started")
      (doall
       (pmap
-       (fn []
-         )
-       (mc/find-maps @db "published" {:type :retweet})))
-     ))
-
-
-  (time
-   (do
-     (say/say "Start computation")
-     (doall
-      (for [{:keys [tweet _id]} (mc/find-maps @db "published" {:type :retweet})]
-        (let [{{:keys [id_str]} :retweeted_status} (mc/find-map-by-id @db "tweets" tweet)
-              {stid :_id} (mc/find-one-as-map @db "tweets" {:id_str id_str})
-              {spid :_id} (if stid (mc/find-one-as-map @db "published" {:tweet stid})
-                              nil)]
-          (store-reaction _id spid))))
-     (say/say "Computation completed!")))
-
-
-  (->> (mc/find-maps @db "published" {:type :retweet})
-       (pmap :tweet)
-       (pmap type)
-       (into #{})
-       aprint)
-
-
-  )
+       (fn [{:keys [tweet _id]}]
+         (let [{aid :article} (mc/find-one-as-map @db "origins" {:tweet tweet})
+               {source-tweet :tweet} (mc/find-one-as-map @db "origins" {:article aid :source {$ne nil}})]
+           (if source-tweet
+             (store-reaction tweet source-tweet)
+             nil)))
+       (mc/find-maps @db "published" {:type :share})))
+     (say/say "Computation completed")))
+
+  (defrecord Publication [source reactions])
+
+  (letfn [(find-reactions [{:keys [_id ts] :as pub}]
+            (let [reactions (mc/find-maps @db "reactions" {:source _id})]
+              (Publication. pid (mapv #(find-reactions (:publication %)) reactions))))
+          (impact [tree]
+            (loop [counter 0
+                   max-path 0
+                   delays []
+                   loc tree]
+              (if (zip/end? loc)
+                {:size counter
+                 :height max-path
+                 :delays}
+                (recur
+                 (if (zip/node loc) (inc counter) counter)
+                 (if (zip/node loc) (-> loc zip/path count (max max-path)) max-path)
+                 (if (zip/node loc)
+                   (let [pub-time (-> (zip/root tree) :pub :ts)
+                         post-delay (if (t/after? (-> loc zip/node :pub :ts) pub-time)
+                                      (t/interval pub-time (-> loc zip/node :pub :created_at))
+                                      (t/interval (-> loc zip/node :pub :created_at) pub-time))]
+           (conj delays (t/in-seconds post-delay)))
+         delays)
+                 (zip/next loc)))))
+          (reaction-tree [{:keys [_id]}]
+            (zip/zipper
+             (fn [node] true)
+             (fn [node] (:reactions node))
+             (fn [node new-children] (assoc-in node [:reactions] new-children))
+             (find-reactions _id)))
+          (avg [coll]
+            (float (/ (reduce + coll) (count coll))))]
+    (let [users (map :_id (mc/find-maps @db "users" {:screen_name {$in news-accounts}}))
+          pubs (take 100 (mc/find-maps @db "published" {:user {$in users}}))
+          reaction-trees (pmap reaction-tree pubs)
+          impacts (pmap impact reaction-trees)]
+      (-> {:avg-size (avg (pmap :size impacts))
+           :avg-height (avg (pmap :height impacts))
+           :overall-size (reduce + (pmap :size impacts))
+           :max-height (apply max (pmap :height impacts))}
+          aprint
+          time)))
+
+
+  (->> (mc/find-maps @db "users")
+       (pmap #(mc/count @db "published" {:user %}))
+       frequencies
+       aprint
+       time)
+
+
+)


CLJS-482: bind *out* as the target of Rhino REPL env printing

--- src/clj/cljs/repl/rhino.clj
+++ src/clj/cljs/repl/rhino.clj
@@ -102,11 +102,11 @@
                         '(ns cljs.user))
     (ScriptableObject/putProperty scope
                                   "out"
-                                  (Context/javaToJS System/out scope))
+                                  (Context/javaToJS *out* scope))
     (repl/evaluate-form repl-env
                         env
                         "<cljs repl>"
-                        '(set! *print-fn* (fn [x] (.print js/out x))))))
+                        '(set! *print-fn* (fn [x] (.write js/out x))))))

 (defrecord RhinoEnv [loaded-libs]
   repl/IJavaScriptEnv


fix stupid bug, it looks like we're finally seeing something resembling round tripping

--- src/clj/cljs/source_map.clj
+++ src/clj/cljs/source_map.clj
@@ -111,7 +111,7 @@
         (fn [info source-idx line col]
           (let [segv [(:gcol info) source-idx line col]]
             (if-let [name (:name info)]
-              (let [idx (if-let [idx (get-in @names->idx name)]
+              (let [idx (if-let [idx (get @names->idx name)]
                           idx
                           (let [cidx @name-idx]
                             (swap! names->idx assoc name cidx)


remove unecessary log

remove unecessary log

--- src/clj/datasplash/core.clj
+++ src/clj/datasplash/core.clj
@@ -122,11 +122,9 @@
                  (log/debugf "Catched Exception %s at runtime with message -> %s => already initialized : %s / candidates for init : %s"
                              (type e#) (.getMessage e#) (into #{} already-required#) (into [] nss#))
                  (if (empty? nss#)
-                   (do
-                     (log/errorf e# "ClojureRuntimeException in [%s]" pt-name#)
-                     (throw (ex-info "Runtime exception intercepted"
-                                     (-> {:hostname (get-hostname)}
-                                         (cond-> pt-name# (assoc :name pt-name#))) e#)))
+                   (throw (ex-info "Runtime exception intercepted"
+                                   (-> {:hostname (get-hostname)}
+                                       (cond-> pt-name# (assoc :name pt-name#))) e#))
                    (let [missings# nss# ;; (remove already-required# nss#)
                          missing-at-start?# (not (empty? (remove required-at-start# nss#)))]
                      (if-not (empty? missings#)


Fixed require and added docstring.

Fixed require and added docstring.

--- src/clj/io/cogitat/boot.clj
+++ src/clj/io/cogitat/boot.clj
@@ -1,8 +1,11 @@
 (ns io.cogitat.boot
-  (:require [boot.core :refer [deftask]]))
+  (:require [tailrecursion.boot.core :refer [deftask]]))

-(deftask foo [boot]
+(deftask foo
+  "A simple task to show the basic form; simply prints a message."
+  [boot]
+  (print
   (fn [continue]
     (fn [event]
       (println "Hello, boot!")


Add basic profiler utilities

Add basic profiler utilities

--- src/cljs/koeeoadi/util.cljs
+++ src/cljs/koeeoadi/util.cljs
@@ -4,6 +4,12 @@
             [goog.array :refer [forEach]]
             [om.next :as om]
             [om.dom :as dom]))
+
+(defn profile-start []
+  (.profile js/console "Profile Run"))
+
+(defn profile-stop []
+  (.profileEnd js/console))

 (defn keycode [e]
   (.-keyCode e))


Increase border strength and correct bug (move strength for new point)

--- src/cljs/triboard/core.cljs
+++ src/cljs/triboard/core.cljs
@@ -25,7 +25,7 @@
 (def init-block-count 12) ;; Init blocks for red, blue, green, gray
 (def players [:blue :red :green])
 (def directions [[-1 0] [1 0] [0 -1] [0 1] [-1 1] [1 1] [-1 -1] [1 -1]])
-
+(def max-score (- (* 16 11) 12))

 ;; -----------------------------------------
 ;; EXTRINSIC TYPES
@@ -52,7 +52,7 @@

 (defn scores? [s]
   (and
-    (<= 0 (reduce + (vals s)) 164)
+    (<= 0 (reduce + (vals s)) max-score)
     (every? s players)))

@@ -236,8 +236,9 @@
   {:pre [(coord? point)]}
   [board point]
   (let [neighbors (coord-neighbors point)
-        walls (filter #(= :wall (get-in board % :wall)) neighbors)]
-    (+ 1 (* (count walls) 0.25))))
+        walls (filter #(= :wall (get-in board % :wall)) neighbors)
+        count-wall (count walls)]
+    (+ 1 (* count-wall count-wall 0.25))))

 (defn with-cells-strength
   "Adds to a given game the strength of each of its cells"
@@ -249,14 +250,16 @@

 (defn- move-strength
   "Compute the strength of a move, based on the converted cells"
-  [cells-strength converted-filter [move converted]]
+  [cells-strength converted-filter [point converted]]
   {:pre [(fn? converted-filter) (every? move? converted)]}
   (transduce
     (comp
       (filter converted-filter) ;; TODO - Extract this part (specific to worst move)
       (mapcat :taken)
       (map cells-strength))
-    + converted))
+    +
+    (cells-strength point)
+    converted))

 (defn- worst-immediate-loss
   "Return the next worse lost game move for 'looser' if 'player' plays"


Update end-of-game message to reflect successful funding

--- src/cljs/zetawar/views.cljs
+++ src/cljs/zetawar/views.cljs
@@ -313,12 +313,11 @@
      [:> js/ReactBootstrap.Modal.Title
       "Congratulations! You won!"]]
     [:> js/ReactBootstrap.Modal.Body
-     "Thanks for playing Zetawar! "
-     "If you enjoyed it and are interested in seeing Zetawar turn into a "
-     "full featured open source game complete with multiplayer, custom maps, "
-     "and programmable AI, please consider supporting the "
-     [:a {:href "https://www.kickstarter.com/projects/311016908/zetawar"}
-      "Zetawar Kickstarter"] "!"]
+     "Thanks for playing Zetawar! If you're interested in staying up-to-date"
+     " with Zetawar as it develops, please follow "
+     [:a {:href "https://twitter.com/ZetawarGame"}
+      "@ZetawarGame"]
+     " on Twitter."]
     [:> js/ReactBootstrap.Modal.Footer
      [:button.btn.btn-default {:on-click #(handlers/hide-win-dialog conn %)}
       "Close"]]]


Typo fixes

Typo fixes

--- src/flatgui/paint.clj
+++ src/flatgui/paint.clj
@@ -384,7 +384,6 @@
 (defn- ready-for-paint? [container]
   (and
     (:position-matrix container)
-    (:clip-size container)
     (:viewport-matrix container)
     (:clip-size container)
     (:look-vec container)))
@@ -398,5 +397,5 @@
       (apply concat
              [(conj id-path (:id container))]
              (for [[_ v] (:children container)] (get-paint-all-sequence (conj id-path (:id container)) v)))
-      container))
+      (throw (IllegalStateException. "No ready for paint"))))
   ([container] (get-paint-all-sequence [] container)))


made id generator bit less dumb

made id generator bit less dumb

--- src/gnostica/core.clj
+++ src/gnostica/core.clj
@@ -30,6 +30,11 @@
    :up    {:x 0 :y 0}})

 (def id-generator (atom 0))                                 ;temp, its a bit silly because you have to increment manually
+
+(defn- get-next-minion-id []
+  (let [ret-val @id-generator]
+    (swap! id-generator inc)
+    ret-val))

 (defn- create-board [size]
   (let [coords (for [x (range size)
@@ -90,7 +95,7 @@
 (defn create [game from source-id new-minion-orientation]
   (let [source-minion (get-in game [:board from :minions source-id])
         to (calculate-target-loc from source-minion)
-        new-minion-id @id-generator
+        new-minion-id (get-next-minion-id)
         verify-validity
         (fn [game]
           (verify-is-using-own-minion game source-minion)
@@ -104,7 +109,6 @@
                                         :orientation new-minion-orientation
                                         :owner (:owner source-minion)})]
     (verify-validity game)
-    (swap! id-generator inc)
     [(end-turn create) new-minion-id]))