A synth is now a reader fn that returns a map with both input and output, which allows better composition of non-singleton synths.

--- src/klangmeister/sound/synthesis.cljs
+++ src/klangmeister/sound/synthesis.cljs
@@ -1,25 +1,31 @@
 (ns klangmeister.sound.synthesis)
+
+; A synth is a function that returns a section.
+(defn section
+  ([input output] {:input input :output output})
+  ([singleton] (section singleton singleton)))

 ; Plumbing

 (defn run-with
-  "Convert a node (actually a reader fn) into a concrete audio node by supplying context and timing."
-  [node context at duration]
-  (node context at duration))
+  "Convert a synth (actually a reader fn) into a concrete section by supplying context and timing."
+  [synth context at duration]
+  (synth context at duration))

 (defn destination [context at duration]
-  (.-destination context))
+  (section (.-destination context)))

 (defn plug [param input context at duration]
-  "Plug an input into an audio parameter, accepting both numbers and nodes."
+  "Plug an input into an audio parameter, accepting both numbers and synths."
   (if (number? input)
     (.setValueAtTime param input at)
-    (-> input (run-with context at duration) (.connect param))))
+    (-> input (run-with context at duration) :output (.connect param))))

 (defn gain [level]
   (fn [context at duration]
-    (doto (.createGain context)
-      (-> .-gain (plug level context at duration)))))
+    (section
+      (doto (.createGain context)
+        (-> .-gain (plug level context at duration))))))

 ; Envelopes
@@ -36,7 +42,7 @@
           (+ dx x))
         at
         corners)
-      audio-node)))
+      (section audio-node))))

 (defn adshr [attack decay sustain hold release]
   (envelope [attack 1.0] [decay sustain] [hold sustain] [release 0]))
@@ -55,26 +61,28 @@
 ; Combinators

 (defn connect
-  "Use the output of one node as the input to another."
-  [node1 node2]
+  "Use the output of one synth as the input to another."
+  [upstream-synth downstream-synth]
   (fn [context at duration]
-    (let [sink (-> node2 (run-with context at duration))]
-      (-> node1 (run-with context at duration) (.connect sink))
-      sink)))
+    (let [{downstream-input :input downstream-output :output} (-> downstream-synth (run-with context at duration))
+          {upstream-input :input upstream-output :output} (-> upstream-synth (run-with context at duration))]
+      (-> upstream-output (.connect downstream-input))
+      (section upstream-input downstream-output))))

 (defn connect->
-  "Connect nodes in series."
+  "Connect synths in series."
   [& nodes]
   (reduce connect nodes))

 (defn add
-  "Add together nodes by connecting them all to the same gain."
-  [& nodes]
+  "Add together synths by connecting them all to the same gain."
+  [& synths]
   (fn [context at duration]
-    (let [sink (-> (gain 1.0) (run-with context at duration))]
-      (doseq [node nodes]
-        (-> node (run-with context at duration) (.connect sink)))
-      sink)))
+    (let [{downstream-input :input downstream-output :output} (-> (gain 1.0) (run-with context at duration))]
+      (doseq [synth synths]
+        (let [{:keys [input output]} (-> synth (run-with context at duration))]
+          (-> output (.connect downstream-input))))
+      (section downstream-input downstream-output))))

 ; Noise
@@ -89,9 +97,10 @@
           data (.getChannelData buffer 0)]
       (doseq [i (range sample-rate)]
         (aset data i (generate-bit!)))
-      (doto (.createBufferSource context)
-        (-> .-buffer (set! buffer))
-        (.start at)))))
+      (section
+        (doto (.createBufferSource context)
+          (-> .-buffer (set! buffer))
+          (.start at))))))

 (def white-noise
   (let [white #(-> (js/Math.random) (* 2.0) (- 1.0))]
@@ -107,12 +116,13 @@
 (defn oscillator
   [type freq]
   (fn [context at duration]
-    (doto (.createOscillator context)
-      (-> .-frequency .-value (set! 0))
-      (-> .-frequency (plug freq context at duration))
-      (-> .-type (set! type))
-      (.start at)
-      (.stop (+ at duration 1.0))))) ; Give a bit extra for the release
+    (section
+      (doto (.createOscillator context)
+        (-> .-frequency .-value (set! 0))
+        (-> .-frequency (plug freq context at duration))
+        (-> .-type (set! type))
+        (.start at)
+        (.stop (+ at duration 1.0)))))) ; Give a bit extra for the release

 (def sine (partial oscillator "sine"))
 (def sawtooth (partial oscillator "sawtooth"))
@@ -125,14 +135,16 @@
 (defn biquad-filter
   ([type freq q]
    (fn [context at duration]
-     (doto (-> (biquad-filter type freq) (run-with context at duration))
-       (-> .-Q (plug q context at duration)))))
+     (section
+       (doto (-> (biquad-filter type freq) (run-with context at duration) :output)
+         (-> .-Q (plug q context at duration))))))
   ([type freq]
    (fn [context at duration]
-     (doto (.createBiquadFilter context)
-       (-> .-frequency .-value (set! 0))
-       (-> .-frequency (plug freq context at duration))
-       (-> .-type (set! type))))))
+     (section
+       (doto (.createBiquadFilter context)
+         (-> .-frequency .-value (set! 0))
+         (-> .-frequency (plug freq context at duration))
+         (-> .-type (set! type)))))))

 (def low-pass (partial biquad-filter "lowpass"))
 (def high-pass (partial biquad-filter "highpass"))
@@ -142,15 +154,17 @@

 (defn stereo-panner [pan]
   (fn [context at duration]
-    (doto (.createStereoPanner context)
-      (-> .-pan (plug pan context at duration)))))
+    (section
+      (doto (.createStereoPanner context)
+        (-> .-pan (plug pan context at duration))))))

 (defn delay-line
   [time]
   (fn [context at duration]
     (let [maximum 5]
-      (doto (.createDelay context maximum)
-        (-> .-delayTime (plug time context at duration))))))
+      (section
+        (doto (.createDelay context maximum)
+          (-> .-delayTime (plug time context at duration)))))))

 ; FIXME: Unify with noise.
 (defn convolver
@@ -162,8 +176,9 @@
           data (.getChannelData buffer 0)]
       (doseq [i (range sample-rate)]
         (aset data i (generate-bit! i)))
-      (doto (.createConvolver context)
-        (-> .-buffer (set! buffer))))))
+      (section
+        (doto (.createConvolver context)
+          (-> .-buffer (set! buffer)))))))

 (def reverb
   (let [duration 5


Fixed missing namespace use.

Fixed missing namespace use.

--- src/asid/calling_card.clj
+++ src/asid/calling_card.clj
@@ -25,9 +25,9 @@
 (defn- http-failed? [resp]
   (let [status (:status resp)]
     (cond
-     (some #{404 406} [status]) (bad-request "Remote endpoint did not understand request.")
-     (< 399 status 500) (unavailable)
-     (> status 500) (bad-gateway)
+     (some #{404 406} [status]) (ed/bad-request "Remote endpoint did not understand request.")
+     (< 399 status 500) (ed/unavailable)
+     (> status 500) (ed/bad-gateway)
      :else resp)))

 (defn- find-letterplate [card]


Update utility & sample in package litedocx

--- src/bakyeono/litedocx.clj
+++ src/bakyeono/litedocx.clj
@@ -43,15 +43,7 @@
              page-width page-height page-orientation
              page-margin-top page-margin-bottom page-margin-left page-margin-right
              page-margin-header page-margin-footer
-             title subject creator keywords description last-modified-by]
-      :or {:application "" :app-version ""
-           :page-width "210mm" :page-height "297mm" :page-orientation "portrait"
-           :page-margin-top 720 :page-margin-bottom 720
-           :page-margin-left 720 :page-margin-right 720
-           :page-margin-header 0 :page-margin-footer 0
-           :title "" :subject ""
-           :creator "" :last-modified-by ""
-           :keywords "" :description ""}}]
+             title subject creator keywords description last-modified-by]}]
   (let [content-types-xml (w/content-types-xml resources)
         rels-rels (w/rels-rels)
         doc-props-app-xml (w/doc-props-app-xml params)
@@ -71,45 +63,65 @@

 (defn sample1
   []
-  (write-docx :styles [(w/paragraph-style "headline"
+  (write-docx "SAMPLE1.docx"
+              :styles [(w/paragraph-style "headline"
                                           :font "Sans Serif"
                                           :font-color "000000"
-                                          :font-size 20
+                                          :font-size 40
                                           :bold true
                                           :align "both")
                        (w/paragraph-style "date"
                                           :font "Arial"
                                           :font-color "4444CC"
-                                          :font-size 12
+                                          :font-size 20
                                           :italics true
-                                          :algin "right")
+                                          :align "right")
                        (w/paragraph-style "author"
                                           :font "Arial"
                                           :font-color "aa0000"
-                                          :font-size 13
+                                          :font-size 24
                                           :algin "right")
                        (w/paragraph-style "monospace"
                                           :font "Monospace"
                                           :font-color "000000"
-                                          :algin "left")
-                       (w/paragraph-style "body")
+                                          :align "left")
+                       (w/paragraph-style "h1"
+                                          :font "Arial"
+                                          :font-color "000000"
+                                          :font-size 24
+                                          :bold true)
+                       (w/paragraph-style "body"
+                                          :font "Arial"
+                                          :font-color "000000"
+                                          :font-size 20)
                        (w/table-style "table"
                                       :font "Arial"
                                       :font-color "449944"
                                       :align "center")]
               ;; TODO: need a better way to create resource meta datas
               :resources (w/resources ["image/png" "foo.png" (load-byte-array "foo.png")])
-              :document [(w/table "a1" :center [964 964 8072]
-                                  (w/tr (w/td 964 (w/p "a" "name"))
-                                        (w/td 964 (w/p "a" "age"))
-                                        (w/td 8072 (w/p "a" "description"))))
-                         (w/p "para1"
-                              "This is a paragraph.")
-                         (w/img "rId2"
-                                "800px"
-                                "600px"
-                                :name "foo.png"
-                                :desc "Image Description: this is a testing image.")]))
+              :body [(w/p "headline" "A Sample Document Written With Litedocx")
+                     (w/p "date" (str (java.util.Date.)))
+                     (w/p "author" "Bak Yeon O")
+                     (w/p "body" "")
+                     (w/p "body" "An easy way to make DOCX documents with Clojure.")
+                     (w/p "monospace" "int main(int argc, char** argv)")
+                     ;; TODO: need a better way to create tables
+                     (w/table "a1" :center [964 964 8072]
+                              (w/tr (w/td 964 (w/p "h1" "name"))
+                                    (w/td 964 (w/p "h1" "age"))
+                                    (w/td 8072 (w/p "h1" "description")))
+                              (w/tr (w/td 964 (w/p "body" "Alexandra"))
+                                    (w/td 964 (w/p "body" "30"))
+                                    (w/td 8072 (w/p "body" "A young woman.")))
+                              (w/tr (w/td 964 (w/p "body" "Mellisa"))
+                                    (w/td 964 (w/p "body" "32"))
+                                    (w/td 8072 (w/p "body" "Another young woman."))))
+                     (w/img "rId2"
+                            "800px"
+                            "600px"
+                            :name "foo.png"
+                            :desc "Image Description: this is a testing image.")]))

 (defn sample-pack
   []


Log exceptions on bad responses from requests to the Visual Studio Online api

--- src/build_mon/vso_api.clj
+++ src/build_mon/vso_api.clj
@@ -9,7 +9,9 @@
                         repository-id "/commits/" source-version "?api-version=1.0")]
     (try (-> (client/get commit-url {:basic-auth ["USERNAME CAN BE ANY VALUE" token]})
              :body (json/parse-string true) :comment)
-         (catch Exception e))))
+         (catch Exception e
+           (prn "Bad Response when attempting to retrieve commit message.")
+           (prn e)))))

 (defn- retrieve-last-two-builds [account project token build-definition-id]
   (let [last-two-builds-url (str "https://" account  ".visualstudio.com/defaultcollection/"
@@ -17,7 +19,9 @@
                                  "&definitions=" build-definition-id)]
     (try (-> (client/get last-two-builds-url {:basic-auth ["USERNAME CAN BE ANY VALUE" token]})
              :body (json/parse-string true) :value)
-         (catch Exception e))))
+         (catch Exception e
+           (prn "Bad Response when attempting to retrieve last two builds.")
+           (prn e)))))

 (defn retrieve-build-info [account project token build-definition-id]
   (let [[build previous-build] (retrieve-last-two-builds account project token build-definition-id)
@@ -31,4 +35,6 @@
                                    project "/_apis/build/definitions?api-version=2.0")]
     (try (-> (client/get build-definitions-url {:basic-auth ["USERNAME CAN BE ANY VALUE" token]})
              :body (json/parse-string true) :value)
-         (catch Exception e))))
+         (catch Exception e
+           (prn "Bad Response when attempting to retrieve build definitions.")
+           (prn e)))))


Handle validation errors where the error is not a map

--- src/catalog/web/views.clj
+++ src/catalog/web/views.clj
@@ -169,13 +169,20 @@
                         [:thead [:tr (map #(vec [:th %]) ["Record-id" "Title" "Field" "Value" "Error"])]]
                         [:tbody
                          (for [[errors {:keys [record-id title] :as item}] problems]
-                           (for [[k v] errors]
+                           (if (map? errors)
+                             (for [[k v] errors]
+                               [:tr
+                                [:td record-id]
+                                [:td (h title)]
+                                [:td k]
+                                [:td (h (item k))]
+                                [:td (str v)]])
                              [:tr
                               [:td record-id]
                               [:td (h title)]
-                              [:td k]
-                              [:td (h (item k))]
-                              [:td (str v)]]))]]
+                              [:td ]
+                              [:td ]
+                              [:td (pr-str errors)]]))]]
                        (form/form-to
                         {:enctype "multipart/form-data"}
                         [:post (format "/upload/%s/%s" year issue)]


Fixed a missing :refer in the ns-declaration.

--- src/chessground/data.cljs
+++ src/chessground/data.cljs
@@ -2,7 +2,7 @@
   "Representation and manipulation of the application data"
   (:require [chessground.common :as common :refer [pp]]
             [chessground.chess :as chess]
-            [chessground.schemas :refer [ChessState]]
+            [chessground.schemas :refer [ChessState Square]]
             [schema.core :as s :refer [Str Bool]])
   (:require-macros [schema.macros :as sm :refer [defschema]]))


stop gravity go-block when quit channel is closed

--- src/client/game/core.cljs
+++ src/client/game/core.cljs
@@ -250,13 +250,24 @@
             speed (if (:soft-drop @state)
                     (min soft-speed level-speed)
                     level-speed)
-
-            cs [(timeout speed) pause-grav] ; channels to listen to (timeout, pause)
-            [_ c] (alts! cs)]               ; get the first channel to receive a value
-        (if (= pause-grav c)                ; if "pause" received, wait for "resume"
-          (<! resume-grav)
-          (apply-gravity!))
-        (recur)))))
+            time-chan (timeout speed)
+            quit-chan (:quit-chan @state)
+            [_ c] (alts! [time-chan pause-grav quit-chan])]
+
+        (condp = c
+
+          pause-grav
+          (let [[_ c] (alts! [resume-grav quit-chan])]
+            (if (= c resume-grav)
+              (recur)))
+
+          time-chan
+          (do
+            (apply-gravity!)
+            (recur))
+
+          nil)))))
+

 ;;------------------------------------------------------------
 ;; Input-driven STATE CHANGES


find article's source tweet

find article's source tweet

--- src/clj/ceres/curator.clj
+++ src/clj/ceres/curator.clj
@@ -224,6 +224,9 @@
   (mc/count (:db @mongo-state) "articles"))

+(defn find-source [id]
+  (mc/find-maps (:db @mongo-state) "urls" {$and [{:article id} {:source {$ne nil}}]} [:source :tweet :ts]))
+
 (comment

   ;; TODO update on server
@@ -253,16 +256,28 @@
                               (into #{})
                               frequencies)))

-  (->> (mc/find-maps (:db @mongo-state) "urls" {:source "SPIEGELONLINE"})
-       (map #(mc/find-maps (:db @mongo-state) "urls" {:article (:article %)}))
-       flatten
-       count)
-
-  (->> (mc/find-maps (:db @mongo-state) "urls")
+  (mc/ensure-index (:db @mongo-state) "articles" (array-map :ts 1))
+
+
+
+  (time
+   (loop [round 10]
+     (when (> round 0)
+
+       (recur (dec round)))))
+
+  (->> (mc/find-maps (:db @mongo-state) "urls" {:ts {$gt (t/date-time 2014 7 26 14)}})
        (map :article)
        frequencies
        (sort-by val >)
        (take 10)
-       (map #(vec [(:title (mc/find-map-by-id (:db @mongo-state) "articles" (key %))) %])))
-
-)
+       )
+
+  (->> (mc/find-maps (:db @mongo-state) "articles" {:ts {$gt (t/today)}} [:title])
+       (take 50)
+       (map #(vec [(:title %) (find-source (:_id %))]))
+       (remove #(empty? (second %)))
+      clojure.pprint/pprint)
+
+
+  )


add spread functionality

add spread functionality

--- src/clj/ceres/curator.clj
+++ src/clj/ceres/curator.clj
@@ -254,6 +254,41 @@
   (mc/find-maps (:db @mongo-state) "urls" {$and [{:article id} {:source {$ne nil}}]} [:source :tweet :ts]))

+(defn spread [tweet]
+  (let [tweet-id (:id_str tweet)
+        neighbor-tweets (->> (mc/find-maps
+                              (:db @mongo-state)
+                              "tweets"
+                              {$and [{$or [{"retweeted_status.id_str" tweet-id}
+                                           {"in_reply_to_status_id_str" tweet-id}]}
+                                     {:created_at {$gt (t/date-time 2014 7 1)}}]}))]
+    {:source (:_id tweet)
+     :reactions (doall (pmap spread neighbor-tweets))}))
+
+(defn compute-impact-graph [origin]
+  (let [article (if (:article origin)
+                      (mc/find-map-by-id (:db @mongo-state) "articles" (:article origin))
+                      nil)
+        tweet (mc/find-map-by-id (:db @mongo-state) "tweets" (:tweet origin))]
+    (let [related-origins (if article
+                             (->> (mc/find-maps (:db @mongo-state) "origins" {:article  (:_id article)} [:tweet])
+                                  (remove #(= (:_id %) (:_id origin)))
+                                  (pmap :tweet)
+                                  (pmap #(mc/find-map-by-id (:db @mongo-state) "tweets" %)))
+                             nil)
+          related-tweets (->> (mc/find-maps
+                                (:db @mongo-state)
+                                "tweets"
+                                {$and [{$or [{"retweeted_status.id_str" (:id_str tweet)}
+                                             {"in_reply_to_status_id_str" (:id_str tweet)}]}
+                                       {:created_at {$gt (t/date-time 2014 7 1)}}]}))
+          merged-articles (into related-origins related-tweets)]
+      (->> {:source-article (:_id article)
+            :source-tweet (:_id tweet)
+            :reactions (pmap spread (into #{} merged-articles))}
+           clojure.pprint/pprint))))
+
+
 (comment

   ;; TODO update on server
@@ -273,32 +308,18 @@

   (mc/ensure-index (:db @mongo-state) "tweets" (array-map :id 1) {:unique true})

-  (->> (mc/find-maps (:db @mongo-state) "urls" {:article (first sz-url)})
-       (mapv #(assoc % :tweet  (mc/find-map-by-id (:db @mongo-state) "tweets" (:tweet %) [:in_reply_to_status_id_str :text :user.screen_name :retweeted :id_str])))
-       clojure.pprint/pprint)
-
-
-  (->> (mc/find-maps (:db @mongo-state) "articles" {:ts {$gt (t/today)}} [:title])
-                  (map #(vec [(:_id %) (:title %) (first (find-source (:_id %)))]))
-                  (remove #(empty? (last %)))
-                  (filter #(= "SPIEGELONLINE" (-> % last :source)))
-                  rand-nth)
-
-  (def spon-url (mc/find-map-by-id (:db @mongo-state) "urls" (ObjectId. "53d615fe657a4f9d852ca271")))
-
-  (def spon-tweet (mc/find-map-by-id (:db @mongo-state) "tweets" (-> spon-url :tweet)))
-
-  spon-url
-  spon-tweet
-
-  (->> (compute-tweet-diffusion (:id_str spon-tweet) []) flatten count)
-
-  (->> (mc/find-maps (:db @mongo-state) "origins" {:article (:article spon-url)})
-       (mapv #(assoc % :tweet  (mc/find-map-by-id (:db @mongo-state) "tweets" (:tweet %) [:in_reply_to_status_id_str :text :user.screen_name :retweeted_status.id_str :id_str])))
-       count)
-
-(->> (mc/find-maps (:db @mongo-state) "urls" {:article (:article spon-url)})
-       (mapv #(assoc % :tweet  (mc/find-map-by-id (:db @mongo-state) "tweets" (:tweet %) [:in_reply_to_status_id_str :text :user.screen_name :retweeted_status.id_str :id_str])))
-       count)
+  (compute-impact-graph (rand-nth (mc/find-maps (:db @mongo-state) "origins" {:source {$in (:news-accounts @mongo-state)}})))
+
+
+  (->> (mc/find-maps (:db @mongo-state) "articles")
+       rand-nth
+       time)
+
+  (mc/count (:db @mongo-state) "origins" {:source {$in (:news-accounts @mongo-state)}})
+
+  (-> (spread spon-tweet)
+      clojure.pprint/pprint)
+
+

   )


5 shards is too many

5 shards is too many

--- src/clj/runbld/schema.clj
+++ src/clj/runbld/schema.clj
@@ -302,7 +302,8 @@
 (def StoredBuildIndexSettings
   {:settings
    {:analysis
-    {:analyzer m/classpath-analyzer}}
+    {:analyzer m/classpath-analyzer}
+    :index {:number_of_shards 2}}
    :mappings
    {DocType
     {:_all {:enabled false}
@@ -433,7 +434,9 @@
                 :version m/multi-string}}}}}}}})

 (def StoredFailureIndexSettings
-  {:mappings
+  {:settings
+   {:index {:number_of_shards 2}}
+   :mappings
    {DocType
     {:_all {:enabled false}
      :properties
@@ -450,7 +453,9 @@
     :total  s/Num}})

 (def StoredLogIndexSettings
-  {:mappings
+  {:settings
+   {:index {:number_of_shards 2}}
+   :mappings
    {DocType
     {:_all {:enabled false}
      :properties


tiny bugfix

tiny bugfix

--- src/clj/tictag/server.clj
+++ src/clj/tictag/server.clj
@@ -124,7 +124,7 @@
                    (partial twilio/valid-sig? twilio)))
    (PUT "/time/:timestamp" _
      (wrap-authenticate
-      (partial handle-timestamp db beeminder)
+      (partial timestamp db beeminder)
       (partial valid-shared-secret? shared-secret)))
    (GET "/pings" _ (pings db))
    (GET "/" _ (index))


fix sin to cos

fix sin to cos

--- src/cljs_flake/flake.cljs
+++ src/cljs_flake/flake.cljs
@@ -23,15 +23,15 @@

 (defn- cos
   [deg]
-  (let [rad (* (/ deg 180) (aget js/Math "PI"))]
-  (.sin js/Math rad)))
+  (let [rad (* (/ deg 180.0) (aget js/Math "PI"))]
+    (.cos js/Math rad)))

 (defn- sector
   [angle len]
   (let [angle-b (- base-angle angle)
-        angle-c (- 120 (* 2 angle))
+        angle-c (- 120 (* 2.0 angle))
         angle-d (- 180 angle)
-        sector-len (/ (/ len 2) (cos angle-b))]
+        sector-len (/ (/ len 2.0) (cos angle-b))]
     [:left angle :fwd sector-len
      :left angle-c :fwd sector-len
      :right angle-d :fwd len]))


Mesh registry

Mesh registry

name conventions

--- src/clojure/yaw/world.clj
+++ src/clojure/yaw/world.clj
@@ -26,7 +26,7 @@
                     :mesh/cone (yaw.mesh/cone-geometry)
                     :mesh/pyramid (yaw.mesh/pyramid-geometry)}})))

-(defn register-mesh
+(defn register-mesh!
   "Given a universe, a keyword id, and mesh data, associates the id to the data in the universe atom"
   [univ id mesh]
   (swap! univ assoc-in [:meshes id] mesh))


Add != to Number instances

Add != to Number instances

--- src/clojure_ruby/core.clj
+++ src/clojure_ruby/core.clj
@@ -67,7 +67,10 @@
                      (create-number (- (:data this) other))))
              "==" (fn number-equal [this other]
                     (let [other (host-send other :number)]
-                      (= (:data this) other)))}
+                      (= (:data this) other)))
+             "!=" (fn number-not-equal [this other]
+                    (let [other (host-send other :number)]
+                      (not= (:data this) other)))}
    :host-methods {:number (fn number-host-number [this] (:data this))}})

 (defn create-string [s]


CI doesn't know about jsx yet

CI doesn't know about jsx yet

--- src/dev/demo/browser.cljs
+++ src/dev/demo/browser.cljs
@@ -10,7 +10,7 @@
     ["./es6.js" :as es6]
     ["./foo" :as foo]
     ["circular-test" :as circ]
-    ["/demo/myComponent" :refer (myComponent)]
+    #_ ["/demo/myComponent" :refer (myComponent)]
     [cljs.test :refer (deftest)]
     ))

@@ -24,7 +24,7 @@

 (es6/someAsyncFn (js/fetch "/index.html"))

-(js/console.log "JSX" (myComponent))
+#_ (js/console.log "JSX" (myComponent))

 (pprint [1 2 3])


change icon names to keywords

change icon names to keywords

Former-commit-id: 74e5c4128d428826da3f092dfdf9ee328391e2c1

--- src/syng_im/components/main-tabs.cljs
+++ src/syng_im/components/main-tabs.cljs
@@ -19,12 +19,12 @@
          :tab-list       [{:handler #(dispatch [:navigate-to
                                                 :chat-list])
                            :title   "Chats"
-                           :icon    "icon_tab_chats"}
+                           :icon    :icon_tab_chats}
                           {:handler #(dispatch [:navigate-to
                                                 :discovery])
                            :title   "Discover"
-                           :icon    "icon_tab_discovery"}
+                           :icon    :icon_tab_discovery}
                           {:handler #(dispatch [:navigate-to
                                                 :contact-list])
                            :title   "Contacts"
-                           :icon    "icon_tab_contacts"}]}])
+                           :icon    :icon_tab_contacts}]}])


Adds length

Adds length

--- src/the_little_clojurian/chapter4.clj
+++ src/the_little_clojurian/chapter4.clj
@@ -160,3 +160,14 @@
   (is (= (divide 15 4) 3))
   (is (= (divide 15 3) 5))
   (is (= (divide 100 10) 10)))
+
+(with-test
+  (def length
+    (fn [lat]
+      (cond (null? lat) 0
+            :else (add1 (length (cdr lat))))))
+
+  (is (= (length '()) 0))
+  (is (= (length '(hotdogs)) 1))
+  (is (= (length '(hotdogs with mustard sauerkraut and pickles)) 6))
+  (is (= (length '(ham and cheese on rye)) 5)))


helper to build the page type db in one simple fun call

--- corpusmaker.clj
+++ corpusmaker.clj
@@ -32,6 +32,8 @@

 (def *pages-db* "entities-pages")
 (def *types-db* "pages-types")
+(def *wikipage-filename* "wikipage_en.nt")
+(def *instancetype-filename* "instancetype_en.nt")

 (defn process-file [filename line-func]
   ; TODO: add start line and end line (max)
@@ -70,6 +72,23 @@
   (redis/with-server
     server-params
     (process-file filename join-page-type)))
+
+(defn build-page-type-db
+  "Build a redis db from dbpedia NT dumps to be found in folder
+
+  Only instances-types_en.nt and wikipage_en.nt are required."
+  [folder server-params]
+  (collect-entities-to-pages
+    (str folder "/" *wikipage-filename*)
+    server-params)
+  (collect-pages-to-types
+    (str folder "/" *instancetype-filename*)
+    server-params))
+
+(comment
+  (use 'corpusmaker)
+  (time (build-page-type-db "/home/ogrisel/data/dbpedia" {}))
+)
 ;;
 ;; Utilities to parse a complete Wikimedia XML dump to extract sentence that
 ;; contain token annotated with a wiki link that point to a page that matches a


Avoid promaturely destracturing method/model in codegen functions

This makes them a little more encapsulated (a calee doesn't need
to know what parts of the model they need).

--- src/gogolica/generate.clj
+++ src/gogolica/generate.clj
@@ -14,20 +14,24 @@
             [camel-snake-kebab.core :refer :all]))

 (defn generate-ns-declaration
-  "Generates the ns declaration for an API.
-  Takes name, version, description and docs link, all strings."
-  [name version desc docs-link]
+  "Generates the ns declaration for the given API model."
+  [{name :name
+    version :version
+    desc :description
+    docs-link :documentationLink}]
   `(~'ns
     ~(symbol (str "gogolica." name "." version)) ;; HACK: instead of having the ns as string, we should probably read it from the current one
-    ~(str desc "\n\nDocumentation link: " docs-link)
+    ~(str desc "\n\n"
+          "Documentation link: " docs-link)
     (:gen-class)
     (:require [gogolica.common :refer [~'?assoc ~'exec-http] :as ~'common]
               [clojure.string :as ~'str])))

-;; TODO move util generation to another function or a dedicated namespace
 (defn generate-global-vars
-  "Generates global variables that are used throughout the generated namespace"
-  [root-url service-path]
+  "Generates global variables that are used throughout the generated
+   namespace for the given API model"
+  [{root-url :rootUrl
+    service-path :servicePath}]
   `[(def ~'base-url ~(str root-url service-path))])

 (defn split-required-params
@@ -49,16 +53,18 @@
        "\n")) ;; TODO: add description for parameters

 (defn generate-args
-  "Given a list of parameter maps and the parameterOrder vector,
-  generates the arguments vector.
+  "Given a method model, uses a list of parameter maps and the parameterOrder vector,
+  to generate the arguments vector.
   If the request is not nil, then the method also takes a request object,
   that gets specified as the first parameter."
-  [parameters parameterOrder request]
+  [{parameters :parameters
+    parameter-order :parameterOrder
+    request :request}]
   (let [[required optional] (->> parameters
                                  split-required-params
                                  (mapv (comp (partial mapv ->kebab-case-symbol) keys)))
-        ;; parameterOrder also contains the required params, so we get them from there
-        required (mapv ->kebab-case-symbol parameterOrder)
+        ;; parameter-order also contains the required params, so we get them from there
+        required (mapv ->kebab-case-symbol parameter-order)
         request-sym (when request
                       (->kebab-case-symbol (get request :$ref)))
         required (if request-sym
@@ -102,7 +108,9 @@
 (defn generate-request
   "Generates the request map to be passed to the http library.
   NB: uses the `base-url` symbol, it should be generated in the ns including the method."
-  [http-method path parameters]
+  [{http-method :httpMethod
+    path :path
+    parameters :parameters}]
   (let [query-params (->> parameters
                           (filter (fn [[_ v]] (= (:location v) "query")))
                           (mapv   (fn [[k _]] (name k))))
@@ -123,12 +131,12 @@

 (defn generate-function-from-method
-  [{:keys [id httpMethod parameters path parameterOrder request] :as method}]
+  [{:keys [id] :as method}]
   `(~'defn ~(generate-function-name id)
      ~(generate-docs method)
-     ~(generate-args parameters parameterOrder request)
-     (~'println ~(generate-request httpMethod path parameters))
-    (~'http/request ~(generate-request httpMethod path parameters))))
+     ~(generate-args method)
+     (~'println ~(generate-request method))
+    (~'http/request ~(generate-request method))))

 (defn pprint-symbols
   [symbols]
@@ -139,10 +147,9 @@
 (defn generate-ns
   "Given a service model, generates a clojure namespace with the implementation
    of all the API methods as functions and returns it as a pretty string."
-  [{:keys [name version description documentationLink rootUrl servicePath]
-    :as model}]
-  (let [ns-declaration (generate-ns-declaration name version description documentationLink)
-        global-vars (generate-global-vars rootUrl servicePath)
+  [model]
+  (let [ns-declaration (generate-ns-declaration model)
+        global-vars (generate-global-vars model)
         functions (->> (:resources model)
                        (mapcat
                         (fn [[resource-id resource]]


test for multi-input payment

test for multi-input payment

--- test/block_chain/integration_test.clj
+++ test/block_chain/integration_test.clj
@@ -175,5 +175,18 @@
            (:public key-a))))))

 (deftest test-generating-payment-from-multiple-inputs)
+(deftest test-generating-payment-from-multiple-inputs
+  (let [chain (atom [])]
+    (miner/mine-and-commit chain (blocks/generate-block [(miner/coinbase pem-a)] {:target easy-difficulty-target :chain @chain}))
+    (miner/mine-and-commit chain (blocks/generate-block [(miner/coinbase pem-a)] {:target easy-difficulty-target :chain @chain}))
+    (let [p (miner/generate-payment key-a pem-b 50 @chain)
+          sig (:signature (first (:inputs p)))]
+      (is (= 2 (count (:inputs p))))
+      (is (= 1 (count (:outputs p))))
+      (is (= 50 (reduce + (map :amount (:outputs p)))))
+      (is (wallet/verify
+           sig
+           (txn/txn-signable p)
+           (:public key-a))))))
 (deftest test-generating-payment-with-change)
 (deftest test-generating-payment-with-transaction-fee)


restructure tests for clarity

restructure tests for clarity

--- test/calfpath/route_prepare_test.cljc
+++ test/calfpath/route_prepare_test.cljc
@@ -16,16 +16,30 @@
         :clj [calfpath.route :as r])))

-(defn handler
-  [ks]
-  (fn [request]
-    (select-keys request (conj ks :request-method))))
+;; ----- utility -----
+
+
+(defn handler [ks] (fn [request]
+                     (select-keys request (conj ks :request-method))))

 (def er-handler (handler [:path-params :uri :method]))

-(def easy-routes
+(defn deep-dissoc [data k] (walk/prewalk (fn [node]
+                                           (if (map? node)
+                                             (dissoc node k)
+                                             node))
+                             data))
+
+
+(defn remove-handler [routes] (deep-dissoc routes :handler))
+
+
+;; ----- easy-route tests -----
+
+
+(def easy-routes1
   [{["/album/:lid/artist/:rid/" :get] er-handler}
    {"/hello/1234/"                    er-handler}
    {["/info/:token/"            :get] er-handler}
@@ -39,32 +53,18 @@
                   {"" er-handler}]}])

-(def flat-routes
-  [{:uri "/info/:token/"            :method :get}
-   {:uri "/album/:lid/artist/:rid/" :method :get}
-   {:uri "/user/:id/profile/:type/" :nested [{:method :get   }
-                                             {:method :patch }
-                                             {:method :delete}]}
-   {:uri "/user/:id/permissions/"   :nested [{:method :get   }
-                                             {:method :post  }
-                                             {:method :put   }]}
-   {:uri "/user/:id/auth"}
-   {:uri "/user/:id"     }
-   {:uri "/hello/1234/"  }])
-
-
-(def tidy-routes
+(def flat-routes1  ; flat-routes are the NON-easy regular version of easy-routes
   [{:uri "/album/:lid/artist/:rid/" :method :get}
    {:uri "/hello/1234/"                         }
    {:uri "/info/:token/"            :method :get}
-   {:uri "/user/:id*" :nested [{:uri "/auth"    }
-                               {:uri "/permissions/"   :nested [{:method :get   }
-                                                                {:method :post  }
-                                                                {:method :put   }]}
-                               {:uri "/profile/:type/" :nested [{:method :get   }
-                                                                {:method :patch }
+   {:uri "/user/:id*" :nested [{:uri "/auth"}
+                               {:uri "/permissions/"   :nested [{:method :get }
+                                                                {:method :post}
+                                                                {:method :put }]}
+                               {:uri "/profile/:type/" :nested [{:method :get}
+                                                                {:method :patch}
                                                                 {:method :delete}]}
-                               {:uri ""         }]}])
+                               {:uri ""}]}])

 (def easy-routes2
@@ -79,41 +79,40 @@
                               {:method :put}]}])

+(deftest test-easy
+  (is (= flat-routes1 (-> easy-routes1
+                        (r/easy-routes :uri :method)
+                        remove-handler)))
+  (is (= flat-routes2 (-> easy-routes2
+                        (r/easy-routes :uri :method)
+                        remove-handler))))
+
+
+;; ----- tidy-route tests -----
+
+
+(def tidy-routes1
+  [{:uri "/album/:lid/artist/:rid/" :method :get}
+   {:uri "/hello/1234/"                         }
+   {:uri "/info/:token/"            :method :get}
+   {:uri "/user/:id*" :nested [{:uri "/auth"    }
+                               {:uri "/permissions/"   :nested [{:method :get   }
+                                                                {:method :post  }
+                                                                {:method :put   }]}
+                               {:uri "/profile/:type/" :nested [{:method :get   }
+                                                                {:method :patch }
+                                                                {:method :delete}]}
+                               {:uri ""         }]}])
+
+
 (def tidy-routes2
   [{:uri "/user*" :nested [{:uri "/:id" :nested [{:method :get}
                                                  {:method :put}]}
                            {:uri "" :method :post}]}])

-(defn deep-dissoc [data k]
-  (walk/prewalk (fn [node]
-                  (if (map? node)
-                    (dissoc node k)
-                    node))
-          data))
-
-
-(deftest test-easy-routes
-  (is (= tidy-routes
-        (deep-dissoc
-          (r/easy-routes easy-routes :uri :method)
-          :handler)))
-  (is (= flat-routes2
-        (deep-dissoc
-          (r/easy-routes easy-routes2 :uri :method)
-          :handler))))
-
-
 (deftest test-routes->wildcard-tidy
-  (is (= tidy-routes
-        (-> flat-routes
-          (r/update-routes r/routes->wildcard-tidy {:tidy-threshold 2}))))
-  (is (= tidy-routes2
-        (-> flat-routes2
-          (r/update-routes r/routes->wildcard-tidy {:tidy-threshold 1})))))
-
-
-(def tidy-easy-routes2
-  [{["/user" :post] (constantly {:status 200 :body "new-user"})}
-   {"/user/:id" [{:get (constantly {:status 200 :body "user-get"})}
-                 {:put (constantly {:status 200 :body "user-put"})}]}])
+  (is (= tidy-routes1 (-> flat-routes1
+                        (r/update-routes r/routes->wildcard-tidy {:tidy-threshold 2}))))
+  (is (= tidy-routes2 (-> flat-routes2
+                        (r/update-routes r/routes->wildcard-tidy {:tidy-threshold 1})))))


:nail_care:

--- test/com/wsscode/pathom/gen_test.cljc
+++ test/com/wsscode/pathom/gen_test.cljc
@@ -116,64 +116,69 @@
   (is (= (sgen/comp->db Component)
          {::fixed-number 42 ::fixed-str "bla"})))

+(defn generate-response
+  ([query] (generate-response {} query))
+  ([env query]
+   (gen/generate (sgen/query-props-generator env query))))
+
 (deftest test-query-data-generator
-  (is (= (gen/generate (sgen/query-props-generator gen-env [::fixed-number ::fixed-str ::undefined]))
+  (is (= (generate-response gen-env [::fixed-number ::fixed-str ::undefined])
          {::fixed-number 42 ::fixed-str "bla" ::undefined nil}))

-  (is (= (gen/generate (sgen/query-props-generator gen-env [::fixed-number ::fixed-str
-                                                            {:simple-join [::fixed-str]}]))
+  (is (= (generate-response gen-env [::fixed-number ::fixed-str
+                                              {:simple-join [::fixed-str]}])
          {::fixed-number 42,
           ::fixed-str    "bla",
           :simple-join   {::fixed-str "bla"}}))

-  (is (= (gen/generate (sgen/query-props-generator gen-env [::fixed-number ::fixed-str
-                                                            {:ui/join [::fixed-number]}]))
+  (is (= (generate-response gen-env [::fixed-number ::fixed-str
+                                              {:ui/join [::fixed-number]}])
          {::fixed-number 42 ::fixed-str "bla"}))

-  (is (= (gen/generate (sgen/query-props-generator {::sgen/settings {::number-list {::sgen/coll 10}}}
-                         [{::number-list [::fixed-number]}]))
+  (is (= (generate-response {::sgen/settings {::number-list {::sgen/coll 10}}}
+           [{::number-list [::fixed-number]}])
          {::number-list (repeat 10 {::fixed-number 42})}))

-  (is (= (gen/generate (sgen/query-props-generator {::sgen/settings {::number-list {::sgen/coll 10
-                                                                                ::sgen/distinct ::fixed-number}}}
-                         [{::number-list [::fixed-number]}]))
+  (is (= (generate-response {::sgen/settings {::number-list {::sgen/coll              10
+                                                                      ::sgen/distinct ::fixed-number}}}
+           [{::number-list [::fixed-number]}])
          {::number-list [{::fixed-number 42}]}))

-  (is (= (gen/generate (sgen/query-props-generator {::sgen/settings {::fixed-number {::sgen/gen (s/gen #{43})}}}
-                         [::fixed-number]))
+  (is (= (generate-response {::sgen/settings {::fixed-number {::sgen/gen (s/gen #{43})}}}
+           [::fixed-number])
          {::fixed-number 43}))

-  (is (= (gen/generate (sgen/query-props-generator [[::fixed-number '_]]))
+  (is (= (generate-response [[::fixed-number '_]])
          {[::fixed-number '_] 42}))

-  (is (= (gen/generate (sgen/query-props-generator [{[::some-id 123]
-                                                     [::some-id ::fixed-str]}]))
+  (is (= (generate-response [{[::some-id 123]
+                                       [::some-id ::fixed-str]}])
          {[:com.wsscode.pathom.gen-test/some-id 123]
           {:com.wsscode.pathom.gen-test/some-id   123
            :com.wsscode.pathom.gen-test/fixed-str "bla"}}))

-  (is (= (gen/generate (sgen/query-props-generator
-                         {::sgen/transform-generator
-                          (fn [x] (gen/fmap inc x))}
-                         [::fixed-number]))
+  (is (= (generate-response
+           {::sgen/transform-generator
+            (fn [x] (gen/fmap inc x))}
+           [::fixed-number])
          {::fixed-number 43}))

   (testing "meta settings"
-    (is (= (gen/generate (sgen/query-props-generator {}
-                           (with-meta [::fixed-number]
-                             {::sgen/settings
-                              {::fixed-number {::sgen/gen (s/gen #{420})}}})))
+    (is (= (generate-response {}
+             (with-meta [::fixed-number]
+               {::sgen/settings
+                {::fixed-number {::sgen/gen (s/gen #{420})}}}))
            {::fixed-number 420}))

-    (is (= (gen/generate (sgen/query-props-generator {}
-                           (with-meta [::fixed-number]
-                             {::sgen/fmap #(update % ::fixed-number inc)})))
+    (is (= (generate-response {}
+             (with-meta [::fixed-number]
+               {::sgen/fmap #(update % ::fixed-number inc)}))
            {::fixed-number 43}))

-    (is (= (gen/generate (sgen/query-props-generator {}
-                           (with-meta [::n1 ::n2]
-                             {::sgen/such-that (fn [{::keys [n1 n2]}]
-                                                 (> n1 n2))})))
+    (is (= (generate-response {}
+             (with-meta [::n1 ::n2]
+               {::sgen/such-that (fn [{::keys [n1 n2]}]
+                                   (> n1 n2))}))
            {::n1 2 ::n2 1}))))

 (deftest test-comp-data-generator


firefox canvas mask attr instead of inline style

--- src-cljs/frontend/components/canvas.cljs
+++ src-cljs/frontend/components/canvas.cljs
@@ -1001,7 +1001,7 @@
                       :id "svg-canvas"
                       :xmlns "http://www.w3.org/2000/svg"
                       :key "svg-canvas"
-                      :style (when (ua-browser/isFirefox) #js {:mask (utils/absolute-css-hash "canvas-mask")})
+                      :mask (when (ua-browser/isFirefox) (utils/absolute-css-hash "canvas-mask"))
                       :className (str "canvas-frame "
                                       (cond (keyboard/arrow-shortcut-active? app) " arrow-tool "
                                             (keyboard/pan-shortcut-active? app) " pan-tool "


clean out old circle icons

clean out old circle icons

--- src-cljs/frontend/components/common.cljs
+++ src-cljs/frontend/components/common.cljs
@@ -106,16 +106,7 @@
    :user {:paths [:user-stroke]}
    :users {:paths [:users-stroke]}
    :bullet {:paths [:dot-stroke]}
-   :logo {:paths [:turn :circle]}
-   :pass {:paths [:turn :check]}
-   :fail {:paths [:turn :times]}
    :queued {:paths [:turn :clock]}
-   :logo-light {:paths [:slim_turn :slim_circle]}
-   :busy-light {:paths [:slim_turn :slim_circle]}
-   :pass-light {:paths [:slim_check]}
-   :fail-light {:paths [:slim_times]}
-   :hold-light {:paths [:slim_clock]}
-   :stop-light {:paths [:slim_ban]}
    :chat {:paths [:chat-stroke]}
    :crosshair {:paths [:crosshair-stroke]}
    :ibeam {:paths [:ibeam-stroke]}
@@ -134,11 +125,7 @@
    :newdoc {:paths [:newdoc-stroke]}
    :login {:paths [:login-stroke]}
    :logout {:paths [:logout-stroke]}
-   :info {:paths [:info-stroke]}
-   :settings-light {:paths [:slim_settings :slim_circle]}
-   :none-light {:paths [:slim_circle]}
-   :repo {:paths [:repo]}
-   :spinner {:paths [:turn :circle]}})
+   :info {:paths [:info-stroke]}})

 (defn svg-icon [icon-name & [{:keys [path-props svg-props]}]]
   (let [template (get icon-templates icon-name)]


better mute icon

better mute icon

--- src-cljs/frontend/components/common.cljs
+++ src-cljs/frontend/components/common.cljs
@@ -213,8 +213,6 @@
    :chat-team   {:paths [:stroke-chat-team]}
    :feedback    {:paths [:stroke-feedback]}

-   :sound-mute  {:paths [:stroke-sound
-                         :stroke-slash-backward]}
    :sound-off   {:paths [:stroke-sound]}
    :sound-min   {:paths [:stroke-sound
                          :stroke-wave1]}
@@ -225,6 +223,11 @@
                          :stroke-wave1
                          :stroke-wave2
                          :stroke-wave3]}
+   :sound-mute  {:paths [:stroke-sound
+                         :stroke-wave1
+                         :stroke-wave2
+                         :stroke-wave3
+                         :stroke-slash-backward]}

    :click       {:paths [:fill-click-cursor
                          :stroke-click-1


remove some inspects

remove some inspects

--- frontend/controllers/post_controls.cljs
+++ frontend/controllers/post_controls.cljs
@@ -118,8 +118,8 @@
         parent (sel1 target "#container_parent")
         parent-scroll-left (.-scrollLeft parent)
         current-container (sel1 target (str "#container_" current-container-id))
-        current-container-scroll-left (inspect (int (.-x (goog.style.getContainerOffsetToScrollInto current-container parent))))
-        parent-scroll-left (inspect (.-scrollLeft parent))
+        current-container-scroll-left (int (.-x (goog.style.getContainerOffsetToScrollInto current-container parent)))
+        parent-scroll-left (.-scrollLeft parent)
         containers (sort-by (fn [c] (Math/abs (- parent-scroll-left (.-x (goog.style.getContainerOffsetToScrollInto c parent)))))
                             (sel parent ".container-view"))
         ;; if we're scrolling left, then we want the container whose rightmost portion is showing


When a form upload contains no file, return nil instead of a 0-length byte array.

--- src/appengine_magic/multipart_params.clj
+++ src/appengine_magic/multipart_params.clj
@@ -35,10 +35,12 @@
                [(.getFieldName i)
                 (if (.isFormField i)
                     (Streams/asString (.openStream i) encoding)
-                    (let [upload-bytes (IOUtils/toByteArray (.openStream i))]
+                    (let [upload-bytes (IOUtils/toByteArray (.openStream i))
+                          size (alength upload-bytes)
+                          upload-bytes (if (zero? size) nil upload-bytes)]
                       {:content-type (.getContentType i)
                        :filename (.getName i)
-                       :size (alength upload-bytes)
+                       :size size
                        :bytes upload-bytes}))])
              (itemiterator-to-seq (.getItemIterator (ServletFileUpload.)
                                                     (:request request))))))


finished rebalance implementation

finished rebalance implementation

--- src/clj/backtype/storm/daemon/nimbus.clj
+++ src/clj/backtype/storm/daemon/nimbus.clj
@@ -7,6 +7,7 @@
   (:import [java.nio.channels Channels WritableByteChannel])
   (:use [backtype.storm bootstrap])
   (:use [backtype.storm.daemon common])
+  (:use [clojure.contrib.def :only [defnk]])
   (:gen-class))

 (bootstrap)
@@ -420,22 +421,28 @@

 ;; public so it can be mocked out
 (defn compute-new-task->node+port [conf storm-id existing-assignment storm-cluster-state callback task-heartbeats-cache scratch?]
-  ;; TODO: implement scratch?
-  (let [available-slots (available-slots conf storm-cluster-state callback)
+  (let [available-slots (available-slots conf storm-cluster-state callback)
+        storm-conf (read-storm-conf conf storm-id)
+        all-task-ids (set (.task-ids storm-cluster-state storm-id))
+
+
+        ;; if scratch?, slots = existing slots + available-slots, reassign-ids = all-ids
+        ;; otherwise, do timeout stuff
         existing-assigned (reverse-map (:task->node+port existing-assignment))
-        storm-conf (read-storm-conf conf storm-id)
-
-        ;; if scratch?, slots = existing slots + available-slots
-        ;; otherwise, do timeout stuff
+        alive-ids (if scratch?
+                    all-task-ids
+                    (set (alive-tasks conf storm-id storm-cluster-state
+                                      all-task-ids (:task->start-time-secs existing-assignment)
+                                      task-heartbeats-cache)))

-        all-task-ids (set (.task-ids storm-cluster-state storm-id))
-        alive-ids (set (alive-tasks conf storm-id storm-cluster-state
-                                    all-task-ids (:task->start-time-secs existing-assignment) task-heartbeats-cache))
         alive-assigned (filter-val (partial every? alive-ids) existing-assigned)
-        alive-node-ids (map first (keys alive-assigned))
+
         total-slots-to-use (min (storm-conf TOPOLOGY-WORKERS)
                                 (+ (count available-slots) (count alive-assigned)))
-        keep-assigned (keeper-slots alive-assigned (count all-task-ids) total-slots-to-use)
+        keep-assigned (if scratch?
+                        {}
+                        (keeper-slots alive-assigned (count all-task-ids) total-slots-to-use))
+
         freed-slots (keys (apply dissoc alive-assigned (keys keep-assigned)))
         reassign-slots (take (- total-slots-to-use (count keep-assigned))
                              (sort-slots (concat available-slots freed-slots)))


Functionize all the data elements

Functionize all the data elements

--- src/clj/comic_reader/sites/manga_fox.clj
+++ src/clj/comic_reader/sites/manga_fox.clj
@@ -4,121 +4,114 @@
             [comic-reader.util :refer [safe-read-string]]
             [clojure.string :as s]))

-(def ^:private
-  comic->url-format "%s%s/")
+(defn comic->url-format []
+  "%s%s/")

-(def ^:private
-  comic-list-selector
+(defn comic-list-selector []
   [:div.manga_list :ul :li :a])

-(def ^:private
-  chapter-number-match-pattern #"/c0*(\d+)/")
+(defn chapter-number-match-pattern []
+  #"/c0*(\d+)/")

-(def ^:private
-  chapter-list-selector
+(defn chapter-list-selector []
   [:div#chapters :ul.chlist :li :div #{:h3 :h4} :a])

-(def ^:private
-  page-normalize-pattern #"^\d+$")
+(defn page-normalize-pattern []
+  #"^\d+$")

-(def ^:private
-  page-normalize-format "%s/%s.html")
+(defn page-normalize-format []
+  "%s/%s.html")

-(def ^:private
-  chapter-number-pattern #"/\d+\.html")
+(defn chapter-number-pattern []
+  #"/\d+\.html")

-(def ^:private
-  page-list-selector
+(defn page-list-selector []
   [:div#top_center_bar :form#top_bar :select.m :option])

-(def ^:private
-  image-selector
+(defn image-selector []
   [:div#viewer :img#image])

-(def ^:private
-  root-url "http://mangafox.me")
+(defn root-url []
+  "http://mangafox.me")

-(def ^:private
-  link-url-normalize identity)
+(defn link-url-normalize []
+  identity)

-(def ^:private
-  link-name-normalize first)
+(defn link-name-normalize []
+  first)

-(def ^:private
-  manga-pattern-match-portion "(.*?)/")
+(defn manga-pattern-match-portion []
+  "(.*?)/")

-(def ^:private
-  manga-list-format "%s/manga/")
+(defn manga-list-format []
+  "%s/manga/")

-(def ^:private
-  manga-url-format "%s/manga/")
+(defn manga-url-format []
+  "%s/manga/")

-(def ^:private
-  manga-url
-  (format manga-url-format root-url))
+(defn manga-url []
+  (format (manga-url-format) (root-url)))

-(def ^:private
-  manga-list-url
-  (format manga-list-format root-url))
+(defn manga-list-url []
+  (format (manga-list-format) (root-url)))

-(def ^:private
-  manga-pattern
-  (re-pattern (str manga-url manga-pattern-match-portion)))
+(defn manga-pattern []
+  (re-pattern (str (manga-url) (manga-pattern-match-portion))))

 (def ^:private
   link->map
-  (util/gen-link->map link-name-normalize
-                      link-url-normalize))
+  (util/gen-link->map (link-name-normalize)
+                      (link-url-normalize)))

 (defn extract-image-tag [html]
-  (scrape/extract-image-tag html image-selector))
+  (scrape/extract-image-tag html (image-selector)))

 (defn gen-extract-pages-list-normalize [base-url]
   (util/html-fn {[name] :content}
-    (if-let [page-number (re-find page-normalize-pattern
+    (if-let [page-number (re-find (page-normalize-pattern)
                                   name)]
       {:name name
-       :url (format page-normalize-format
+       :url (format (page-normalize-format)
                     base-url
                     page-number)})))

 (defn extract-pages-list [html chapter-url]
   (let [base-url (s/replace chapter-url
-                            chapter-number-pattern
+                            (chapter-number-pattern)
                             "")
         normalize (gen-extract-pages-list-normalize base-url)]
     (scrape/extract-list html
-                         page-list-selector
+                         (page-list-selector)
                          normalize)))

 (def ^:private chapter-link-normalize
   (comp
    (util/gen-add-key-from-url :ch-num
-                              chapter-number-match-pattern
+                              (chapter-number-match-pattern)
                               safe-read-string)
    link->map))

 (defn extract-chapters-list [html comic-url]
   (scrape/extract-list html
-                       chapter-list-selector
+                       (chapter-list-selector)
                        chapter-link-normalize))

 (def ^:private comic-link-normalize
   (comp
    (util/gen-add-key-from-url :id
-                              manga-pattern)
+                              (manga-pattern))
    link->map))

 (defn extract-comics-list [html]
   (scrape/extract-list html
-                       comic-list-selector
+                       (comic-list-selector)
                        comic-link-normalize))

 (defn comic->url [comic-id]
-  (format comic->url-format manga-url comic-id))
+  (format (comic->url-format) (manga-url) comic-id))

 (defn get-comic-list []
-  (-> manga-list-url
+  (-> (manga-list-url)
       scrape/fetch-url
       extract-comics-list))


make regex go fast

make regex go fast

--- src/com/billpiel/sayid/string_output.clj
+++ src/com/billpiel/sayid/string_output.clj
@@ -560,9 +560,11 @@
              (when b'
                {(->kw b') b'})))))

+(def regex1  (re-pattern (str "\33" "\\[(\\d+)?(;(\\d+))?m")))
+
 (defn ansi->prop-map
   [ansi]
-  (let [rm (re-matcher (re-pattern (str "\33" "\\[(\\d+)?(;(\\d+))?m"))
+  (let [rm (re-matcher regex1
                        ansi)]
     (loop [[match c1 _ c2] (re-find rm)
            agg {}]
@@ -573,9 +575,11 @@
                                              c2)))
         {:text-color agg}))))

+(def regex2 (re-pattern (str "([^" "\33" "]*)(("  "\33" ".*?m)*)")))
+
 (defn str->ansi-pairs
   [s]
-  (let [r (re-pattern (str "([^" "\33" "]*)(("  "\33" ".*?m)+)"))]
+  (let [r regex2]
     (loop [head []
            tail s]
       (let [[match txt ansi _] (re-find r tail)
@@ -608,8 +612,8 @@
        flatten
        (remove nil?)
        group-meta-strings
-       (map #(if (string? %)
-               (str->ansi-pairs %)
+       (mapv #(if (string? %)
+                (str->ansi-pairs %)
                %))
        flatten
        split-text-tag-coll))


Fix multi-args setup on prepare environment

--- src/com/wsscode/pathom/connect/test.cljc
+++ src/com/wsscode/pathom/connect/test.cljc
@@ -376,13 +376,17 @@

 (declare console-print-reporter)

-(defn prepare-environment [{::p.connect/keys [indexes] :as env}]
+(defn prepare-environment [{::p.connect/keys [indexes]
+                            ::keys [data-bank]
+                            :as env}]
   (assert (s/valid? (s/keys :req [::p.connect/indexes]) env)
     (s/explain-str (s/keys :req [::p.connect/indexes]) env))
-  (-> (merge {::data-bank (atom {})
-              ::report-fn console-print-reporter} env)
-      (update ::p.connect/indexes expand-output-tree)
-      (assoc ::multi-args (collect-multi-args indexes))))
+  (let [data-bank (if data-bank
+                    (do (swap! data-bank assoc ::multi-args (collect-multi-args indexes)))
+                    (atom {::multi-args (collect-multi-args indexes)}))]
+    (-> (merge {::data-bank data-bank
+                ::report-fn console-print-reporter} env)
+        (update ::p.connect/indexes expand-output-tree))))

 (s/fdef prepare-environment
   :args (s/cat :env (s/keys :req [::p.connect/indexes]))


Popover HTML should be built after rates are calculated

--- src/main/hbase_region_inspector/core.clj
+++ src/main/hbase_region_inspector/core.clj
@@ -164,8 +164,9 @@
                             :requests-rate (diff-fn % :requests)
                             :write-requests-rate (diff-fn % :write-requests)
                             :read-requests-rate (diff-fn % :read-requests)
-                            :compaction ((juxt :compacted-kvs :total-compacting-kvs) %)
-                            :html (build-html %)) new-regions)]
+                            :compaction ((juxt :compacted-kvs :total-compacting-kvs) %))
+                         new-regions)
+        new-regions (map #(assoc % :html (build-html %)) new-regions)]
     (reset! cached {:updated-at now
                     :regions new-regions})))


In flight.  The sqs listener now works well for multiple ids.

--- src/acyclic/awstools/core.clj
+++ src/acyclic/awstools/core.clj
@@ -1,75 +1,72 @@
 (ns acyclic.awstools.core
   (:use [amazonica.core]
         [amazonica.aws.ec2]
-        acyclic.utils.pinhole)
+        [amazonica.aws.sqs]
+        acyclic.utils.pinhole
+        acyclic.utils.log)
   (:require [clj-ssh.ssh :as ssh]
             [taoensso.timbre :as timbre]
             [clojure.data.codec.base64 :as b64]
+            [clojure.data.json :as json]
+            [clojure.core.async.impl.protocols :as pimpl]
             [clojure.core.async :as async
-             :refer [<! >! <!! >!! timeout chan alt!! go close!]]))
+             :refer [<! >! <!! >!! timeout chan alt!! go close!]])
+  (:import (java.util UUID)))
 (timbre/refer-timbre)

-; ami-5cd51634
-
-

 (def cred (read-string (slurp "AWS.clj")))

 (apply defcredential (map cred [:access-key :secret-key :endpoint]))

 (defn s->b64 [s] (String. (b64/encode (.getBytes s))))
+
+(defn b64->s [s] (String. (b64/decode (.getBytes s))))
+
+(defn- m->pair [[_ k v]] [(keyword k) v])
+
+(defn b64->ec2-data [s]
+  (let [xs  (-> s b64->s clojure.string/split-lines)
+        es  (map #(->> %
+                       (re-matches #"^([\w\-]+):\s*(.*)")
+                       m->pair) xs)]
+    (into {} es)))
+
+(defn chan->seq
+  "Drains a channel onto a lazy sequence.  Blocks internally."
+  [c & [terminate?]]
+  (lazy-seq
+   (when-let [v (if terminate?
+                  (first (async/alts!! [c] :default nil))
+                  (<!! c))]
+     (cons v (chan->seq c terminate?)))))

 (def ^{:private true} paths {:zone  [:launch-specification :placement :availability-zone]
                              :itype [:launch-specification :instance-type]
+                             :subnet [:launch-specification :instance-network-interface-specification 0 :subnet-id]
+                             :group [:launch-specification :network-interfaces 0 :groups 0]
+                             :public? [:launch-specification :network-interfaces 0 :associate-public-ip-address]
                              :price [:spot-price]
                              :n     [:instance-count]
                              :udata [s->b64 :launch-specification :user-data]})

-(defmacro go-try [& forms]
-  `(go (try (do ~@forms)
-            (catch Exception e#
-              (do (debug "Caught exception" e#) nil)))))
-
-(defn patience
-  "Invoke (genfn) every time-wait sec until predicate returns true or time-limit sec has passed.
-Returns result of genfn, or nil in the case of timeout."
-  [genfn pred failure time-limit time-wait]
-  (let [t1     (timeout (* 1000 time-limit))
-        out    (chan)]
-    (async/go-loop []
-      (let [rc     (go-try (genfn))
-            [v c]  (async/alts! [rc t1 out])]
-        (debug "Ats:" v c)
-        (cond
-         (= c out)                    (do (debug "Canceled")
-                                          (when failure (failure)))
-         (= c t1)                     (do (debug time-limit "sec expired")
-                                          (when failure (failure))
-                                          (close! out))
-         (and (= c rc) v (pred v))    (do (debug "Returning" v)
-                                          (>! out v))
-         :else                        (do (debug "Still" v "Waiting" time-wait)
-                                          (<! (timeout (* 1000 time-wait)))
-                                          (debug "Woke up after wait...")
-                                          (recur)))))
-    out))
-
-
-
-(defn pluck [c]
-  (let [a (atom nil)]
-    (close! (go (let [x (<! c)]
-                  (reset! a x))))
-    a))
-
-
-(defn request-spots [req]
-  (let [req    (apply concat (seq req))
-        _      (debug req)
-        rs     (apply request-spot-instances req)]
-    (map :spot-instance-request-id  (:spot-instance-requests rs))))
+(def my-ec2-info nil)
+(def my-req nil)
+(def my-sns-topic nil)
+(def my-sqs-url nil)
+(def my-region "us-east-1")
+(defn slurp-ec2-info [f]
+  (let [i (read-string (slurp f))]
+    (alter-var-root (var acyclic.awstools.core/my-ec2-info) #(do % i))
+    (alter-var-root (var acyclic.awstools.core/my-req) #(do % (:req i)))
+    (alter-var-root (var acyclic.awstools.core/my-region) #(do % (:region i)))
+    (alter-var-root (var acyclic.awstools.core/my-sns-topic) #(do % (:sns-topic i)))
+    (alter-var-root (var acyclic.awstools.core/my-sqs-url) #(do % (:sqs-url i))))
+  nil
+  )
+

 (defn cmd-timeout [sec cmd & args]
   (let [cc (go (apply cmd args))
@@ -78,12 +75,60 @@
     (if (= c cc) v
       (do (debug "cmd" args "timed out after" sec) nil))))

+(defn sqs-listen
+  "Listen on an SQS queue, sending messages to a channel, which is returned."
+ [& [url]]
+  (let [url (or url my-sqs-url)
+        c   (chan)]
+    (async/go-loop []
+      (if (pimpl/closed? c)
+        (debug "Shutting down sqs-listen" url)
+        (let [_ (trace "sqs-listen polling" url)
+              messages  (:messages (receive-message :queue-url url :wait-time-seconds 20))]
+          (trace "sq-listen" url "received" messages)
+          (doseq [{r :receipt-handle b :body} messages]
+            (debug "sqs-listen received" b)
+            (try (>! c (get (json/read-str b) "Message"))
+                 (catch Exception e (info "sqs-listen" (stack-trace e))))
+            (try (delete-message :queue-url url :receipt-handle r)
+                 (catch Exception e (info "sqs-listen" (stack-trace e)))))
+          (recur))))
+    c))
+
+(defmacro go-try [& forms]
+  `(go (try (do ~@forms)
+            (catch Exception e#
+              (do (debug "Caught exception" e#) nil)))))
+
 (defn request-status [rs]
   (let [d    (cmd-timeout 5 describe-spot-instance-requests :spot-instance-request-ids rs)
         sirs (:spot-instance-requests d)
         _    (debug "Request statuses" rs d sirs)]
     (map :state sirs)))

+(defn instance-info-map
+  "Returns map of instance-id to map of information about instance."
+  [is]
+  (let [res   (:reservations (describe-instances :instance-ids is))
+        ims   (map #(get-in % [:instances 0]) res)
+        xms   (map (fn [im] (m-section im
+                      [:instance-id
+                       [:state [:state :name]]
+                       [:ip :private-ip-address]
+                       [:host [:public-dns-name #(when (seq %) %)]]])) ims)]
+    (mseq->m xms :instance-id)))
+
+(defn request-info-map [ids]
+  (let [descs  (describe-spot-instance-requests :spot-instance-request-ids ids)
+        sirs   (:spot-instance-requests descs)
+        sirs   (map (fn [rm] (m-section rm [[:request-id :spot-instance-request-id] :state :instance-id])) sirs)
+        is     (filter string? (map :instance-id sirs))
+        ims    (instance-info-map is)
+        sirs   (map #(merge % (ims (:instance-id %))) sirs)]
+    (mseq->m sirs :request-id)))
+
+
+

 (defn good-strings [vs]
   (and (sequential?  vs)
@@ -94,65 +139,105 @@
   (let [d (describe-spot-instance-requests :spot-instance-request-ids rs)]
     (map :instance-id (:spot-instance-requests d))))

-(defn cancel [rs]
-  (let [c (:cancelled-spot-instance-requests
-        (cancel-spot-instance-requests :spot-instance-request-ids rs))]
-    (debug c)
-    c))
+(defn request-spots [req & opts]
+  (let [req    (apply ph-assoc req paths opts)
+        args   (apply concat (seq req))
+        _      (debug "Spot request" (pr-str args))
+        rs     (apply request-spot-instances args)]
+    (map :spot-instance-request-id  (:spot-instance-requests rs))))

 (defn terminate [is]
   (let [t (terminate-instances :instance-ids is)]
     (debug t)
     t))

-(defn- give-up-and-cancel [req & [msg]]
-  (when msg (debug msg))
-  (concat  (some-> req :is terminate)
-           (some-> req :rs cancel)))
-
-(defn request-spots [req]
-  (let [args    (apply concat (seq req))
-        rs     (apply request-spot-instances args)]
-    (debug "Spot request" args "->" rs)
-    (map :spot-instance-request-id  (:spot-instance-requests rs))))
-
-(defn- add-requests [req]
-  (let [req (assoc req :rs (request-spots (:req req)))]
-    (patience
-     #(assoc req :req-status (cmd-timeout 5 request-status (:rs req)))
-     (fn [req] (let [rss (:req-status req)]
-                 (and (seq rss) (every? #(= "active" %) rss))))
-   #(give-up-and-cancel req "Failed to fulfill spot requests")
-   600 60)))
-
-(defn- add-instances [req]
-  (patience
-   #(assoc req :is (request-instances (:rs req)))
-   #(good-strings (:is %))
-   #(give-up-and-cancel req "Failed to get instance ids")
-   10 1))
-
-
-(defn- add-hosts [req]
-  (patience
-   (fn [] (assoc req :hosts (->> (:is req)
-                                 (describe-instances :instance-ids)
-                                 :reservations
-                                 (map #(get-in % [:instances 0 :public-dns-name])))))
-   #(good-strings (:hosts %))
-   #(give-up-and-cancel req "Failed to get hosts.")
-   10 1))
-
-
-(defn- add-ips [req]
-  (patience
-   (fn [] (assoc req :ips (->> (:is req)
-                                 (describe-instances :instance-ids)
-                                 :reservations
-                                 (map #(get-in % [:instances 0 :private-ip-address])))))
-   #(good-strings (:ips %))
-   #(give-up-and-cancel req "Failed to get IPs")
-   10 1))
+(defn stop [is]
+  (let [t (stop-instances :instance-ids is)]
+    (debug t)
+    t))
+
+(defn cancel-requests [rs]
+  (let [ds (:spot-instance-requests (describe-spot-instance-requests :spot-instance-request-ids rs))
+        is (filter (complement nil?) (map :instance-id ds))
+        cr (:cancelled-spot-instance-requests (cancel-spot-instance-requests :spot-instance-request-ids rs))
+        ci (and (seq is) (terminate is))]
+    [cr ci]))
+
+
+(def ids->chs (atom {}))
+
+(defn notify-chan [id]
+  (let [c (chan)]
+    (swap! ids->chs assoc id c)
+    c))
+
+(defn close-notify-chan! [id]
+  (swap! ids->chs #(let [c (get % id)]
+                     (close! c)
+                     (dissoc % id)))
+  nil)
+
+
+(defn start-up-listener []
+  (let [ctl  (chan)
+        cl   (sqs-listen)]
+    (swap! ids->chs #(assoc % :sqs-listen cl))
+    (async/go-loop []
+      (let [[v c] (async/alts! [ctl cl])]
+        (cond
+         (= c ctl) (do (reset! ids->chs {}) (close! cl))
+         (= c cl)  (do (try
+                          (let [s   (b64->ec2-data v)
+                                id  (:id s)
+                                c   (get @ids->chs id)]
+                            (debug "Got" id c s)
+                            (when c (>! c s)))
+                          (catch Exception e (info e)))
+                       (recur)))))
+    ctl))
+
+
+;;aws --region us-east-1 sns publish --topic-arn  arn:aws:sns:us-east-1:633840533036:instance-up --message yowsassl
+
+(defn send-up [id & {topic :topic region :region}]
+  (str
+   "aws --region " (or region my-region)
+   " sns publish --topic-arn " (or topic my-sns-topic)
+   " --message `(echo \"id: " id "\";bin/ec2-metadata -i -p -o) | base64 -w 0`"))
+
+;(stop-instances :instance-ids ["i-7c949193"])
+;(modify-instance-attribute :instance-id "i-7c949193" :user-data (s->b64 "echo hello"))
+
+
+(defn bring-up-spots  [reqmap nmin cmds & opts]
+  (let [cmd (str (send-up) "\n "(clojure.string/join "\n" cmds))
+        req (apply ph-assoc reqmap paths opts)
+        req (ph-assoc req paths :udata cmd)
+        ;req (assoc req :nmin nmin)
+        n   (ph-get reqmap paths :n)
+        req (if (<= nmin n) req (ph-assoc req paths :n nmin))
+        id  (.toString (UUID/randomUUID))
+        _   (debug "Requesting:" id req)
+        cl  (notify-chan id)
+        rs  (request-spots req)
+        to  (timeout (* 5 60 1000))]
+    (debug "Spot requests:" rs)
+    (async/go-loop [i 0]
+      (if (>= i nmin)
+        (let [rs->info (request-info-map rs)
+              rs-up    (filter rs->info rs)
+              rs-dn    (filter (complement rs->info) rs)]
+          (close-notify-chan! id)
+          (cancel-requests rs-dn)
+          (map rs->info rs-up))
+        (let [[v c] (async/alts! [cl to])]
+          (cond
+           (= c to) (do (info "Timeout on spot request:" req)
+                      (close-notify-chan! id)
+                      (cancel-requests rs)
+                      nil)
+           (= c cl) (recur (inc i))))))))
+

 (def ag (ssh/ssh-agent {}))

@@ -164,12 +249,14 @@
             (ssh/connected? sess) sess
             (zero? n) (do (debug "Failed to connect to" host) nil)
             :else     (do
-                        (ssh/connect sess)
+                        (try (ssh/connect sess)
+                             (catch Exception e (debug "Attempt" n "failed:" e)))
                         (if (ssh/connected? sess)
-                          sess
                           (do
-                            (<! (timeout 1000))
-                            (recur (dec n))))))))))
+                            sess)
+                          (do
+                           (<! (timeout 1000))
+                           (recur (dec n))))))))))

 (defn ssh-sessions [hosts]
@@ -186,55 +273,6 @@

-(defn- add-sessions [req]
-  (let [sess (map #(ssh/session ag % {:strict-host-key-checking :no
-                                      :username "ec2-user"})
-                  (:hosts req))]
-    (patience
-     (fn [] (let [sbad (filter #(not (ssh/connected? %)) sess)]
-              (doseq [s sbad] (ssh/connect s))
-              (assoc req :sessions sess)))
-     #(every? ssh/connected? (:sessions %1))
-     nil ;#(give-up-and-cancel req "Failed to add sesions")
-     60 10
-     )))
-
-(defn chain-some
-  "Each cfn must be a function of one parameter, returning a channel that will deliver
-its non-nil result, or nil on failure.  Non-nil return values will be progressively passed
-to each cfn in order.  Returns a channel that will contain the result of the final
-cfn or nil, if one of them failed along the way."
-  [x & cfns]
-  (let [out (chan)]
-    (debug "Starting" cfns)
-    (async/go-loop [[f & fs] cfns
-                    x        x]
-      (if-not f
-        (do (debug "Finished" cfns)
-            (>! out x))
-        (let [_ (debug "Invoking" f x)
-              y (<! (f x))]
-          (if-not y
-            (do (debug "Failed at" f)
-                (close! out))
-            (recur fs y)))))
-    out))
-
-
-(defn bring-up-aws
-  "Bring up n AWS t1.micro instances and return channel that will contain a single map
-of {:instance ids :hosts names and :sessions objects}, with appropriate timeouts.
-Returns a channel that will contain a map of :hosts, :sessions, etc or nil."
-  [reqmap nmin & opts]
-  (let [req (apply ph-assoc reqmap paths opts)
-        req (assoc req :nmin nmin)
-        n    (ph-get reqmap paths :n)
-        req (if (<= nmin n) req (ph-assoc req paths :n nmin))]
-    (debug "Bringing up" req)
-    (chain-some {:req req} add-requests add-instances add-ips add-hosts)
-    ))
-;; add-sessions
-
 (defn EDNify
 "EDN-ify an arbitrary object, leaving it alone if it's an innocuous string."
   [x]
@@ -246,7 +284,7 @@
 (defn commandify
   "If cmd is a sequence, convert it into a space-delimited string, EDNifying as necessary."
   [cmd]
-  (cond (string? cmd) cmd
+  (cond (string? cmd) (clojure.string/trim cmd)
         (seq cmd) (clojure.string/join " " (map EDNify cmd))))

 (defn ex
@@ -392,3 +430,154 @@
           (give-up-and-cancel res out "Requests failed.")
 )))
     out))
+
+
+#_(defn request-info [ids]
+  (let [d      (cmd-timeout 5 describe-spot-instance-requests :spot-instance-request-ids (filter #(re-find #"^sir-" %) ids))
+        sirs   (:spot-instance-requests d)
+        rs     (map :spot-instance-request-id sirs)
+        rs->rs (into {} (map #(vector %1 %2) rs0 rs))
+        ss     (map :state sirs)
+        is     (map :instance-id sirs)
+        rs->i  (into {} (map #(vector %1 %2) rs is))
+        is-up  (filter identity is)
+        is-r   (:reservations (describe-instances :instance-ids is-up))
+        is-i   (map #(get-in % [:instances 0]) is-r)
+        ;ip-pub (map :public-ip-address is-i)
+        is-up  (map :instance-id is-i)
+        ip-pri (map :private-ip-address is-i)
+        hosts  (map :public-dns-name is-i)
+        hosts  (map #(when (seq %) %) hosts)
+        i->inf (into {} (map #(vector %1 {:instance %1 :ip %2 :host %3}) is-up ip-pri hosts))
+        ]
+    (map #(merge {:request %1 :status %2} ((comp i->inf rs->i) %1)) rs0 ss)))
+
+
+#_(defn patience
+  "Invoke (genfn) every time-wait sec until predicate returns true or time-limit sec has passed.
+Returns result of genfn, or nil in the case of timeout."
+  [genfn pred failure time-limit time-wait]
+  (let [t1     (timeout (* 1000 time-limit))
+        out    (chan)]
+    (async/go-loop []
+      (let [rc     (go-try (genfn))
+            [v c]  (async/alts! [rc t1 out])]
+        (debug "Ats:" v c)
+        (cond
+         (= c out)                    (do (debug "Canceled")
+                                          (when failure (failure)))
+         (= c t1)                     (do (debug time-limit "sec expired")
+                                          (when failure (failure))
+                                          (close! out))
+         (and (= c rc) v (pred v))    (do (debug "Returning" v)
+                                          (>! out v))
+         :else                        (do (debug "Still" v "Waiting" time-wait)
+                                          (<! (timeout (* 1000 time-wait)))
+                                          (debug "Woke up after wait...")
+                                          (recur)))))
+    out))
+
+
+#_(defn pluck [c]
+  (let [a (atom nil)]
+    (close! (go (let [x (<! c)]
+                  (reset! a x))))
+    a))
+
+
+#_(defn- add-sessions [req]
+  (let [sess (map #(ssh/session ag % {:strict-host-key-checking :no
+                                      :username "ec2-user"})
+                  (:hosts req))]
+    (patience
+     (fn [] (let [sbad (filter #(not (ssh/connected? %)) sess)]
+              (doseq [s sbad] (ssh/connect s))
+              (assoc req :sessions sess)))
+     #(every? ssh/connected? (:sessions %1))
+     nil ;#(give-up-and-cancel req "Failed to add sesions")
+     60 10
+     )))
+
+#_(defn chain-some
+  "Each cfn must be a function of one parameter, returning a channel that will deliver
+its non-nil result, or nil on failure.  Non-nil return values will be progressively passed
+to each cfn in order.  Returns a channel that will contain the result of the final
+cfn or nil, if one of them failed along the way."
+  [x & cfns]
+  (let [out (chan)]
+    (debug "Starting" cfns)
+    (async/go-loop [[f & fs] cfns
+                    x        x]
+      (if-not f
+        (do (debug "Finished" cfns)
+            (>! out x))
+        (let [_ (debug "Invoking" f x)
+              y (<! (f x))]
+          (if-not y
+            (do (debug "Failed at" f)
+                (close! out))
+            (recur fs y)))))
+    out))
+
+
+;;
+#_(defn bring-up-aws
+  "Bring up n AWS t1.micro instances and return channel that will contain a single map
+of {:instance ids :hosts names and :sessions objects}, with appropriate timeouts.
+Returns a channel that will contain a map of :hosts, :sessions, etc or nil."
+  [reqmap nmin & opts]
+  (let [req (apply ph-assoc reqmap paths opts)
+        req (assoc req :nmin nmin)
+        n    (ph-get reqmap paths :n)
+        req (if (<= nmin n) req (ph-assoc req paths :n nmin))]
+    (debug "Bringing up" req)
+    (chain-some {:req req} add-requests; add-instances add-ips add-hosts
+                )
+    ))
+;; add-sessions
+
+
+
+#_(defn- add-requests [req]
+  (let [req (assoc req :rs (request-spots (:req req)))]
+    (patience
+     #(assoc req :req-status (cmd-timeout 5 request-status (:rs req)))
+     (fn [req] (let [rss (:req-status req)]
+                 (and (seq rss) (every? #(= "active" %) rss))))
+   #(give-up-and-cancel req "Failed to fulfill spot requests")
+   600 60)))
+
+#_(defn- add-instances [req]
+  (patience
+   #(assoc req :is (request-instances (:rs req)))
+   #(good-strings (:is %))
+   #(give-up-and-cancel req "Failed to get instance ids")
+   10 1))
+
+
+#_(defn- add-hosts [req]
+  (patience
+   (fn [] (assoc req :hosts (->> (:is req)
+                                 (describe-instances :instance-ids)
+                                 :reservations
+                                 (map #(get-in % [:instances 0 :public-dns-name])))))
+   #(good-strings (:hosts %))
+   #(give-up-and-cancel req "Failed to get hosts.")
+   10 1))
+
+
+#_(defn- add-ips [req]
+  (patience
+   (fn [] (assoc req :ips (->> (:is req)
+                                 (describe-instances :instance-ids)
+                                 :reservations
+                                 (map #(get-in % [:instances 0 :private-ip-address])))))
+   #(good-strings (:ips %))
+   #(give-up-and-cancel req "Failed to get IPs")
+   10 1))
+
+#_(defn- give-up-and-cancel [req & [msg]]
+  (when msg (debug msg))
+  (concat  (some-> req :is terminate)
+           (some-> req :rs cancel)))
+


added get angry

added get angry

--- src/babel/english/lexicon.clj
+++ src/babel/english/lexicon.clj
@@ -431,6 +431,18 @@
    "game" {:synsem {:cat :noun
                     :sem {:pred :game
                           :games true}}}
+   "get angry"
+   (let [subject-semantics (ref {:human true})]
+     {:synsem {:cat :verb
+               :sem {:pred :get-angry
+                     :subj subject-semantics
+                     :obj subject-semantics}
+               :subcat {:1 {:sem subject-semantics}
+                        :2 '()}}
+      :english {:participle "getting angry"
+                :present {:3sing "gets angry"}
+                :past "got angry"}})
+
    "get dressed"
    (let [subject-semantics (ref {:human true})]
      {:synsem {:cat :verb


constrain 'be called' to be only present and progressive

--- src/babel/english/lexicon.clj
+++ src/babel/english/lexicon.clj
@@ -103,7 +103,8 @@
                     {:intransitivize false
                      :transitivize false
                      :synsem {:agr subj-agr
-                              :sem {:pred :be-called
+                              :sem {:aspect :progressive
+                                    :pred :be-called
                                     :tense :present
                                     :subj the-real-subj
                                     :obj the-obj}


fix typo

fix typo

--- src/babel/espanol/lexicon.clj
+++ src/babel/espanol/lexicon.clj
@@ -107,7 +107,7 @@
    "desear" {:synsem {:cat :verb
                       :sem {:pred :desire}}}

-   "dormir" {:espanol {:boot-stem1 "duerm"
+   "dormir" {:espanol {:boot-stem1 "durm"
                        :boot-stem2 "dorm"}
              :synsem {:cat :verb
                       :sem {:pred :dormire}}}


whitespace and comments

whitespace and comments

--- src/babel/espanol/lexicon.clj
+++ src/babel/espanol/lexicon.clj
@@ -220,7 +220,6 @@

    "desear" {:synsem {:cat :verb
                       :sem {:pred :desire}}}
-
    "divertirse"
    (let [subject-semantics (ref {:human true})
          subject-agr (ref :top)]
@@ -241,7 +240,6 @@
                        :pret-stem "durm"}
              :synsem {:cat :verb
                       :sem {:pred :sleep}}}
-
    "dormirse"
    (let [subject-semantics (ref {:human true})
          subject-agr (ref :top)]
@@ -718,7 +716,8 @@
                    :pred :I}
              :subcat '()}}})

-;; see TODOs in lexiconfn/compile-lex (should be more of a pipeline as opposed to a argument-position-sensitive function.
+;; see TODOs in lexiconfn/compile-lex (should be more of a pipeline as opposed to a
+;; argument-position-sensitive function.
 (def lexicon
   (future (-> (compile-lex lexicon-source morph/exception-generator morph/phonize)

@@ -743,7 +742,8 @@
                        {:synsem {:subcat {:3 '()}}})

               ;; Cleanup functions can go here. Number them for ease of reading.
-              ;; 1. this filters out any verbs without an inflection: infinitive verbs should have inflection ':infinitive',
+              ;; 1. this filters out any verbs without an inflection: infinitive verbs
+              ;; should have inflection ':infinitive',
               ;; rather than not having any inflection.
               (map-function-on-map-vals
                (fn [k vals]


grammar: s/small/medium/

grammar: s/small/medium/

--- src/babel/francais/writer.clj
+++ src/babel/francais/writer.clj
@@ -67,7 +67,7 @@
                                                                      (process [{:fill-one-language
                                                                                {:count 1
                                                                                :spec spec
-                                                                                :model small
+                                                                                :model medium
                                                                                }}]
                                                                               "fr")
                                                                      (catch Exception e


tweak config spec

tweak config spec

:modules should only accept keyword keys to catch typos like

  :modules {:foo {:init-fn foo.bar/init}}

--- src/main/shadow/build/targets/shared.clj
+++ src/main/shadow/build/targets/shared.clj
@@ -90,16 +90,19 @@
   (s/coll-of keyword? :kind set?))

 (s/def ::module
-  (s/keys
-    :opt-un
-    [::entries
-     ::entry
-     ::init-fn
-     ::depends-on
-     ::prepend
-     ::prepend-js
-     ::append-js
-     ::append]))
+  (s/and
+    ;; {init-fn foo.bar/init} should fail
+    (s/map-of keyword? any?)
+    (s/keys
+      :opt-un
+      [::entries
+       ::entry
+       ::init-fn
+       ::depends-on
+       ::prepend
+       ::prepend-js
+       ::append-js
+       ::append])))

 (s/def ::modules
   (s/map-of


Send proper content types when serving JPGs, PNGs and stylesheets.

The send-static fn in the browser repl tries to detect the content
type of a file by it's pathname extension. This patch adds support for
JPGs, PNGs and stylesheets as well. Without proper content type Chrome
and Firefox are refusing to apply stylesheets.

--- src/clj/cljs/repl/browser.clj
+++ src/clj/cljs/repl/browser.clj
@@ -69,8 +69,12 @@
                                  (str x path)))]
         (server/send-and-close conn 200 (slurp (first local-path))
                         (condp #(.endsWith %2 %1) path
+                          ".html" "text/html"
+                          ".css" "text/css"
+                          ".html" "text/html"
+                          ".jpg" "image/jpeg"
                           ".js" "text/javascript"
-                          ".html" "text/html"
+                          ".png" "image/png"
                           "text/plain"))
         (server/send-404 conn path)))
     (server/send-404 conn path)))


upgrade to clojure 1.2

upgrade to clojure 1.2

--- src/clj/elephantdb/client.clj
+++ src/clj/elephantdb/client.clj
@@ -1,5 +1,5 @@
 (ns elephantdb.client
-  (:use [clojure.contrib.seq-utils :only [shuffle includes?]])
+  (:use [clojure.contrib.seq-utils :only [includes?]])
   (:use [elephantdb thrift hadoop config types util log])
   (:require [elephantdb [shard :as shard]])
   (:import [elephantdb.generated ElephantDB$Iface WrongHostException DomainNotFoundException DomainNotLoadedException])


Renamed download-supervisor to start-download-supervisor and wrap it in a future.

--- src/clj/elephantdb/loader.clj
+++ src/clj/elephantdb/loader.clj
@@ -92,8 +92,11 @@
           (reset! downloaded-kb 0)
           (recur))))))

-(defn download-supervisor [amount-domains max-kbs]
-  (reset! finished-loaders 0)
-  (if (> max-kbs 0) ;; only monitor if there's an actual download throttle
-    (supervise-downloads amount-domains max-kbs)
-    (reset! do-download true)))
+(defn start-download-supervisor [amount-domains max-kbs]
+  (future
+    (log-message "Starting download supervisor")
+    (reset! finished-loaders 0)
+    (if (> max-kbs 0) ;; only monitor if there's an actual download throttle
+      (supervise-downloads amount-domains max-kbs)
+      (reset! do-download true))
+    (log-message "Download supervisor finished")))


Cards enabled

Cards enabled

--- src/clj/funstructor/cards.clj
+++ src/clj/funstructor/cards.clj
@@ -21,6 +21,9 @@
     :action-equality-2
     :action-equality-3

+    :duration-luck-1
+    :duration-luck-2
+
     })

 (def cards


Allow multiple consumers of stream-seq to close.

--- src/clj/http/async/client.clj
+++ src/clj/http/async/client.clj
@@ -220,7 +220,10 @@
       ((fn gen-next []
          (lazy-seq
           (let [v (.take b)]
-            (when-not (= ::done v)
+            (if (= ::done v)
+              (do
+                (.put b ::done)
+                nil)
               (cons v (gen-next)))))))
       b)))


fix create-client's :idle-in-pool-timeout option not being set correctly

--- src/clj/http/async/client.clj
+++ src/clj/http/async/client.clj
@@ -77,7 +77,7 @@
       (when-not (nil? compression-enabled) (.setCompressionEnabled b compression-enabled))
       (when connection-timeout (.setConnectionTimeoutInMs b connection-timeout))
       (when-not (nil? follow-redirects) (.setFollowRedirects b follow-redirects))
-      (when idle-in-pool-timeout (.setConnectionTimeoutInMs b idle-in-pool-timeout))
+      (when idle-in-pool-timeout (.setIdleConnectionInPoolTimeoutInMs b idle-in-pool-timeout))
       (when-not (nil? keep-alive) (.setAllowPoolingConnection b keep-alive))
       (when max-conns-per-host (.setMaximumConnectionsPerHost b max-conns-per-host))
       (when max-conns-total (.setMaximumConnectionsTotal b max-conns-total))


Fix changesets retrieval (see https://bugzilla.mozilla.org/show_bug.cgi?id=1219769 for details)

--- src/clj/medusa/changesets.clj
+++ src/clj/medusa/changesets.clj
@@ -5,13 +5,13 @@
 ;; This library finds changesets given build dates (dates of the form YYYY-MM-DD)
 ;; or build IDs (exact build timestamps of the form YYYYMMDDhhmmss).
 ;;
-;; Example: obtaining a link to the changesets for build date 2015-05-03:
+;; Example: obtaining a link to the changesets for build date 2016-01-06:
 ;;
-;;     (println (find-build-changeset (find-date-buildid "2015-05-03" "mozilla-central") "mozilla-central"))
+;;     (println (find-build-changeset (find-date-buildid "2016-01-06" "mozilla-central") "mozilla-central"))
 ;;
-;; Example: obtaining a link to changesets for buildid 20150503030209
+;; Example: obtaining a link to changesets for buildid 20160106030225
 ;;
-;;     (println (find-build-changeset "20150503030209" "mozilla-central"))
+;;     (println (find-build-changeset "20160106030225" "mozilla-central"))

 (declare elements-by-tag-name)

@@ -42,9 +42,9 @@
   [build-dir-url]
   (let [response (tagsoup/parse build-dir-url)
         links (elements-by-tag-name response :a)
-        text-file-links (filter #(re-find #"^firefox.*win32\.txt$" (get % 2)) links)]
+        text-file-links (filter #(re-find #"^firefox-.*win32\.txt$" (get % 2)) links)]
     (assert (= (count text-file-links) 1) "Could not find revision ID text file")
-    (let [revision-file-url (str build-dir-url (:href (second (first text-file-links))))
+    (let [revision-file-url (str "https://archive.mozilla.org" (:href (second (first text-file-links))))
           revision-file (:body (client/get revision-file-url))
           revision (re-find #"https:\/\/hg\.mozilla.*rev/([0-9a-f]+)$" revision-file)]
       (second revision))))
@@ -64,7 +64,7 @@

         ;; Obtain a list of links to build directories in the desired channel
         build-dirs-url (format "https://archive.mozilla.org/pub/mozilla.org/firefox/nightly/%02d/%02d/" (:y p) (:m p))
-        target (format "%02d-%02d-%02d-%02d-%02d-%02d-%s/" (:y p) (:m p) (:d p) (:hour p) (:min p) (:sec p) channel)
+        target (format "%04d-%02d-%02d-%02d-%02d-%02d-%s/" (:y p) (:m p) (:d p) (:hour p) (:min p) (:sec p) channel)
         response (tagsoup/parse build-dirs-url)
         links (elements-by-tag-name response :a)
         build-dirs-suffix (format "-%s/" channel)
@@ -84,8 +84,15 @@
               build-dir-url (str prev-build-dirs-url (:href (second target-link)))]
           (find-build-dir-revision build-dir-url))
         (let [target-link (nth build-dirs-links target-link-index) ; get the build directory and the revision from the link
-              build-dir-url (str build-dirs-url (:href (second target-link)))]
-          (find-build-dir-revision build-dir-url)))))
+              build-dir-url (str "https://archive.mozilla.org" (:href (second target-link)))]
+          ;; Try to obtain the revision for this build dir, or use the previous build dir if there's no revision in this one
+          ;; This is guaranteed to terminate since we recurse only on build dirs in a given month dir
+          (try
+            (find-build-dir-revision build-dir-url)
+            (catch AssertionError e ; Invalid build dir with no revision, just skip over it and go to the previous build dir link
+              (let [[_ year month day hour minute second] (re-find #"(\d{4})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-(\d{2})-[^/]+/" build-dir-url)
+                    preceding-buildid (str year month day hour minute second)]
+                   (find-preceding-build-revision preceding-buildid channel))))))))

 (defn find-build-changeset
   "Finds the changesets corresponding to the build with buildid `buildid`, which is a URL for a page with a list of all new changesets that went into that build."


Use one-time event

Use one-time event

--- src/clj/puertorico/server.clj
+++ src/clj/puertorico/server.clj
@@ -83,39 +83,47 @@
 (defmethod transition :default [etype state & _]
   state)

-(defn calc-state []
-  (let [cstream @estream]
-    (reduce
-      (fn [state [etype & eargs]]
-        (let [transitioned (transition etype state eargs)
-              checked-turn
-              (if (and (:activerole state) (= (:actionturns state) (dec (:nplayers state))))
-                (-> transitioned
-                    (assoc :actionturns 0 :activerole nil :actionpicker nil)
-                    (update-in [:rolepicker] next-player transitioned))
-                transitioned)
-              checked-fields
-              (if (and (= :startgame etype) (empty? (:fieldtiles checked-turn)))
-                (let [random-fields (common/randomize-fields checked-turn)]
-                  (doseq [ftype random-fields]
-                    (swap! estream conj [:fieldtile ftype] [:field :bank ftype -1]))
-                  checked-turn)
-                checked-turn)
-              ]
-          checked-fields))
-      {:order []
-       :nplayers 0
-       :actionturns 0
-       :activerole nil
-       :fieldtiles []
-       :bank {:vp 0
-              :field-count 0
-              :building nil}}
-        cstream)))
+(defn calc-state [cstream]
+  (reduce
+    (fn [state [etype & eargs]]
+      (transition etype state eargs))
+    {:order []
+     :nplayers 0
+     :actionturns 0
+     :activerole nil
+     :fieldtiles []
+     :bank {:vp 0
+            :field-count 0
+            :building nil}}
+      cstream))
+
+; (let [
+;               checked-turn
+;               (if (and (:activerole state) (= (:actionturns state) (dec (:nplayers state))))
+;                 (-> transitioned
+;                     (assoc :actionturns 0 :activerole nil :actionpicker nil)
+;                     (update-in [:rolepicker] next-player transitioned))
+;                 transitioned)
+;               checked-fields
+;               (if (and (= :startgame etype) (empty? (:fieldtiles checked-turn)))
+;                 (let [random-fields (common/randomize-fields checked-turn)]
+;                   (doseq [ftype random-fields]
+;                     (swap! estream conj [:fieldtile ftype] [:field :bank ftype -1]))
+;                   checked-turn)
+;                 checked-turn)
+;               ]
+;           checked-fields)

 (defn get-players [state]
   (into {}
         (remove (fn [[k v]] (keyword? k)) state)))
+
+
+(defn check-fields [state]
+  (if (zero? (count (:fieldtiles state)))
+    (let [random-fields (common/randomize-fields state)]
+      (doseq [ftype random-fields]
+        (swap! estream conj [:fieldtile ftype] [:field :bank ftype -1])))))

 (defn create-game
   ([p1 p2 p3]
@@ -181,7 +189,9 @@

 (def connections (atom #{}))

-
+(defn estream-handler [k r os ns]
+  (println k))
+
 (defn ws-handler [req]
   (with-channel req channel
     (on-close channel (fn [status]
@@ -189,12 +199,15 @@
                         (println "channel closed")))
     (swap! connections conj channel)

-    (send! channel (pr-str (calc-state)))
+    (add-watch estream :estream estream-handler)
+
+    (send! channel (pr-str (calc-state @estream)))

     (on-receive channel (fn [data]
                           (let [parsed (edn/read-string data)
-                                new-state (calc-state)
-                                new-state (calc-state)]
+                                new-state (calc-state @estream)
+                                cf (check-fields new-state)
+                                new-state (calc-state @estream)]
                             (if (= :reset parsed)
                               (reset-game)
                               (swap! estream conj parsed))