uncomments tests

uncomments tests

--- test/clj/backtype/storm/integration_test.clj
+++ test/clj/backtype/storm/integration_test.clj
@@ -77,520 +77,520 @@
       )))

-;; (deftest test-basic-topology
-;;   (doseq [zmq-on? [true false]]
-;;     (with-simulated-time-local-cluster [cluster :supervisors 4
-;;                                         :daemon-conf {STORM-LOCAL-MODE-ZMQ zmq-on?}]
-;;       (let [topology (thrift/mk-topology
-;;                       {"1" (thrift/mk-spout-spec (TestWordSpout. true) :parallelism-hint 3)}
-;;                       {"2" (thrift/mk-bolt-spec {"1" ["word"]} (TestWordCounter.) :parallelism-hint 4)
-;;                        "3" (thrift/mk-bolt-spec {"1" :global} (TestGlobalCount.))
-;;                        "4" (thrift/mk-bolt-spec {"2" :global} (TestAggregatesCounter.))
-;;                        })
-;;             results (complete-topology cluster
-;;                                        topology
-;;                                        :mock-sources {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}
-;;                                        :storm-conf {TOPOLOGY-DEBUG true
-;;                                                     TOPOLOGY-WORKERS 2})]
-;;         (is (ms= [["nathan"] ["bob"] ["joey"] ["nathan"]]
-;;                  (read-tuples results "1")))
-;;         (is (ms= [["nathan" 1] ["nathan" 2] ["bob" 1] ["joey" 1]]
-;;                  (read-tuples results "2")))
-;;         (is (= [[1] [2] [3] [4]]
-;;                (read-tuples results "3")))
-;;         (is (= [[1] [2] [3] [4]]
-;;                (read-tuples results "4")))
-;;         ))))
-;;
-;; (defbolt identity-bolt ["num"]
-;;   [tuple collector]
-;;   (emit-bolt! collector (.getValues tuple) :anchor tuple)
-;;   (ack! collector tuple))
-;;
-;; (deftest test-system-stream
-;;   ;; this test works because mocking a spout splits up the tuples evenly among the tasks
+(deftest test-basic-topology
+  (doseq [zmq-on? [true false]]
+    (with-simulated-time-local-cluster [cluster :supervisors 4
+                                        :daemon-conf {STORM-LOCAL-MODE-ZMQ zmq-on?}]
+      (let [topology (thrift/mk-topology
+                      {"1" (thrift/mk-spout-spec (TestWordSpout. true) :parallelism-hint 3)}
+                      {"2" (thrift/mk-bolt-spec {"1" ["word"]} (TestWordCounter.) :parallelism-hint 4)
+                       "3" (thrift/mk-bolt-spec {"1" :global} (TestGlobalCount.))
+                       "4" (thrift/mk-bolt-spec {"2" :global} (TestAggregatesCounter.))
+                       })
+            results (complete-topology cluster
+                                       topology
+                                       :mock-sources {"1" [["nathan"] ["bob"] ["joey"] ["nathan"]]}
+                                       :storm-conf {TOPOLOGY-DEBUG true
+                                                    TOPOLOGY-WORKERS 2})]
+        (is (ms= [["nathan"] ["bob"] ["joey"] ["nathan"]]
+                 (read-tuples results "1")))
+        (is (ms= [["nathan" 1] ["nathan" 2] ["bob" 1] ["joey" 1]]
+                 (read-tuples results "2")))
+        (is (= [[1] [2] [3] [4]]
+               (read-tuples results "3")))
+        (is (= [[1] [2] [3] [4]]
+               (read-tuples results "4")))
+        ))))
+
+(defbolt identity-bolt ["num"]
+  [tuple collector]
+  (emit-bolt! collector (.getValues tuple) :anchor tuple)
+  (ack! collector tuple))
+
+(deftest test-system-stream
+  ;; this test works because mocking a spout splits up the tuples evenly among the tasks
+  (with-simulated-time-local-cluster [cluster]
+      (let [topology (thrift/mk-topology
+                      {"1" (thrift/mk-spout-spec (TestWordSpout. true) :p 3)}
+                      {"2" (thrift/mk-bolt-spec {"1" ["word"] ["1" "__system"] :global} identity-bolt :p 1)
+                       })
+            results (complete-topology cluster
+                                       topology
+                                       :mock-sources {"1" [["a"] ["b"] ["c"]]}
+                                       :storm-conf {TOPOLOGY-WORKERS 2})]
+        (is (ms= [["a"] ["b"] ["c"] ["startup"] ["startup"] ["startup"]]
+                 (read-tuples results "2")))
+        )))
+
+(deftest test-shuffle
+  (with-simulated-time-local-cluster [cluster :supervisors 4]
+    (let [topology (thrift/mk-topology
+                    {"1" (thrift/mk-spout-spec (TestWordSpout. true) :parallelism-hint 4)}
+                    {"2" (thrift/mk-bolt-spec {"1" :shuffle} (TestGlobalCount.)
+                                            :parallelism-hint 6)
+                     })
+          results (complete-topology cluster
+                                     topology
+                                     ;; important for test that
+                                     ;; #tuples = multiple of 4 and 6
+                                     :mock-sources {"1" [["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ["a"] ["b"]
+                                                       ]}
+                                     )]
+      (is (ms= (apply concat (repeat 6 [[1] [2] [3] [4]]))
+               (read-tuples results "2")))
+      )))
+
+(defbolt lalala-bolt1 ["word"] [[val :as tuple] collector]
+  (let [ret (str val "lalala")]
+    (emit-bolt! collector [ret] :anchor tuple)
+    (ack! collector tuple)
+    ))
+
+(defbolt lalala-bolt2 ["word"] {:prepare true}
+  [conf context collector]
+  (let [state (atom nil)]
+    (reset! state "lalala")
+    (bolt
+      (execute [tuple]
+        (let [ret (-> (.getValue tuple 0) (str @state))]
+                (emit-bolt! collector [ret] :anchor tuple)
+                (ack! collector tuple)
+                ))
+      )))
+
+(defbolt lalala-bolt3 ["word"] {:prepare true :params [prefix]}
+  [conf context collector]
+  (let [state (atom nil)]
+    (bolt
+      (prepare [_ _ _]
+        (reset! state (str prefix "lalala")))
+      (execute [{val "word" :as tuple}]
+        (let [ret (-> (.getValue tuple 0) (str @state))]
+          (emit-bolt! collector [ret] :anchor tuple)
+          (ack! collector tuple)
+          )))
+    ))
+
+(deftest test-clojure-bolt
+  (with-simulated-time-local-cluster [cluster :supervisors 4]
+    (let [nimbus (:nimbus cluster)
+          topology (thrift/mk-topology
+                      {"1" (thrift/mk-spout-spec (TestWordSpout. false))}
+                      {"2" (thrift/mk-bolt-spec {"1" :shuffle}
+                                              lalala-bolt1)
+                       "3" (thrift/mk-bolt-spec {"1" :shuffle}
+                                              lalala-bolt2)
+                       "4" (thrift/mk-bolt-spec {"1" :shuffle}
+                                              (lalala-bolt3 "_nathan_"))}
+                      )
+          results (complete-topology cluster
+                                     topology
+                                     :mock-sources {"1" [["david"]
+                                                       ["adam"]
+                                                       ]}
+                                     )]
+      (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "2")))
+      (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "3")))
+      (is (ms= [["david_nathan_lalala"] ["adam_nathan_lalala"]] (read-tuples results "4")))
+      )))
+
+(defbolt punctuator-bolt ["word" "period" "question" "exclamation"]
+  [tuple collector]
+  (if (= (:word tuple) "bar")
+    (do
+      (emit-bolt! collector {:word "bar" :period "bar" :question "bar"
+                            "exclamation" "bar"})
+      (ack! collector tuple))
+    (let [ res (assoc tuple :period (str (:word tuple) "."))
+           res (assoc res :exclamation (str (:word tuple) "!"))
+           res (assoc res :question (str (:word tuple) "?")) ]
+      (emit-bolt! collector res)
+      (ack! collector tuple))))
+
+(deftest test-map-emit
+  (with-simulated-time-local-cluster [cluster :supervisors 4]
+    (let [topology (thrift/mk-topology
+                      {"words" (thrift/mk-spout-spec (TestWordSpout. false))}
+                      {"out" (thrift/mk-bolt-spec {"words" :shuffle}
+                                              punctuator-bolt)}
+                      )
+          results (complete-topology cluster
+                                     topology
+                                     :mock-sources {"words" [["foo"] ["bar"]]}
+                                     )]
+      (is (ms= [["foo" "foo." "foo?" "foo!"]
+                ["bar" "bar" "bar" "bar"]   ] (read-tuples results "out"))))))
+
+
+(defn ack-tracking-feeder [fields]
+  (let [tracker (AckTracker.)]
+    [(doto (feeder-spout fields)
+       (.setAckFailDelegate tracker))
+     (fn [val]
+       (is (= (.getNumAcks tracker) val))
+       (.resetNumAcks tracker)
+       )]
+    ))
+
+(defbolt branching-bolt ["num"]
+  {:params [amt]}
+  [tuple collector]
+  (doseq [i (range amt)]
+    (emit-bolt! collector [i] :anchor tuple))
+  (ack! collector tuple))
+
+(defbolt agg-bolt ["num"] {:prepare true :params [amt]}
+  [conf context collector]
+  (let [seen (atom [])]
+    (bolt
+      (execute [tuple]
+        (swap! seen conj tuple)
+        (when (= (count @seen) amt)
+          (emit-bolt! collector [1] :anchor @seen)
+          (doseq [s @seen]
+            (ack! collector s))
+          (reset! seen [])
+          )))
+      ))
+
+(defbolt ack-bolt {}
+  [tuple collector]
+  (ack! collector tuple))
+
+(deftest test-acking
+  (with-tracked-cluster [cluster]
+    (let [[feeder1 checker1] (ack-tracking-feeder ["num"])
+          [feeder2 checker2] (ack-tracking-feeder ["num"])
+          [feeder3 checker3] (ack-tracking-feeder ["num"])
+          tracked (mk-tracked-topology
+                   cluster
+                   (topology
+                     {"1" (spout-spec feeder1)
+                      "2" (spout-spec feeder2)
+                      "3" (spout-spec feeder3)}
+                     {"4" (bolt-spec {"1" :shuffle} (branching-bolt 2))
+                      "5" (bolt-spec {"2" :shuffle} (branching-bolt 4))
+                      "6" (bolt-spec {"3" :shuffle} (branching-bolt 1))
+                      "7" (bolt-spec
+                            {"4" :shuffle
+                            "5" :shuffle
+                            "6" :shuffle}
+                            (agg-bolt 3))
+                      "8" (bolt-spec {"7" :shuffle} (branching-bolt 2))
+                      "9" (bolt-spec {"8" :shuffle} ack-bolt)}
+                     ))]
+      (submit-local-topology (:nimbus cluster)
+                             "acking-test1"
+                             {TOPOLOGY-DEBUG true}
+                             (:topology tracked))
+      (.feed feeder1 [1])
+      (tracked-wait tracked 1)
+      (checker1 0)
+      (.feed feeder2 [1])
+      (tracked-wait tracked 1)
+      (checker1 1)
+      (checker2 1)
+      (.feed feeder1 [1])
+      (tracked-wait tracked 1)
+      (checker1 0)
+      (.feed feeder1 [1])
+      (tracked-wait tracked 1)
+      (checker1 1)
+      (.feed feeder3 [1])
+      (tracked-wait tracked 1)
+      (checker1 0)
+      (checker3 0)
+      (.feed feeder2 [1])
+      (tracked-wait tracked 1)
+      (checker1 1)
+      (checker2 1)
+      (checker3 1)
+
+      )))
+
+(deftest test-ack-branching
+  (with-tracked-cluster [cluster]
+    (let [[feeder checker] (ack-tracking-feeder ["num"])
+          tracked (mk-tracked-topology
+                   cluster
+                   (topology
+                     {"1" (spout-spec feeder)}
+                     {"2" (bolt-spec {"1" :shuffle} identity-bolt)
+                      "3" (bolt-spec {"1" :shuffle} identity-bolt)
+                      "4" (bolt-spec
+                            {"2" :shuffle
+                             "3" :shuffle}
+                             (agg-bolt 4))}))]
+      (submit-local-topology (:nimbus cluster)
+                             "test-acking2"
+                             {}
+                             (:topology tracked))
+      (.feed feeder [1])
+      (tracked-wait tracked 1)
+      (checker 0)
+      (.feed feeder [1])
+      (tracked-wait tracked 1)
+      (checker 2)
+      )))
+
+(defbolt dup-anchor ["num"]
+  [tuple collector]
+  (emit-bolt! collector [1] :anchor [tuple tuple])
+  (ack! collector tuple))
+
+(deftest test-acking-self-anchor
+  (with-tracked-cluster [cluster]
+    (let [[feeder checker] (ack-tracking-feeder ["num"])
+          tracked (mk-tracked-topology
+                   cluster
+                   (topology
+                     {"1" (spout-spec feeder)}
+                     {"2" (bolt-spec {"1" :shuffle} dup-anchor)
+                      "3" (bolt-spec {"2" :shuffle} ack-bolt)}))]
+      (submit-local-topology (:nimbus cluster)
+                             "test"
+                             {}
+                             (:topology tracked))
+      (.feed feeder [1])
+      (tracked-wait tracked 1)
+      (checker 1)
+      (.feed feeder [1])
+      (.feed feeder [1])
+      (.feed feeder [1])
+      (tracked-wait tracked 3)
+      (checker 3)
+      )))
+
+;; (defspout ConstantSpout ["val"] {:prepare false}
+;;   [collector]
+;;   (Time/sleep 100)
+;;   (emit-spout! collector [1]))
+
+;; (def errored (atom false))
+;; (def restarted (atom false))
+
+;; (defbolt local-error-checker {} [tuple collector]
+;;   (when-not @errored
+;;     (reset! errored true)
+;;     (println "erroring")
+;;     (throw (RuntimeException.)))
+;;   (when-not @restarted (println "restarted"))
+;;   (reset! restarted true))
+
+;; (deftest test-no-halt-local-mode
 ;;   (with-simulated-time-local-cluster [cluster]
-;;       (let [topology (thrift/mk-topology
-;;                       {"1" (thrift/mk-spout-spec (TestWordSpout. true) :p 3)}
-;;                       {"2" (thrift/mk-bolt-spec {"1" ["word"] ["1" "__system"] :global} identity-bolt :p 1)
-;;                        })
-;;             results (complete-topology cluster
-;;                                        topology
-;;                                        :mock-sources {"1" [["a"] ["b"] ["c"]]}
-;;                                        :storm-conf {TOPOLOGY-WORKERS 2})]
-;;         (is (ms= [["a"] ["b"] ["c"] ["startup"] ["startup"] ["startup"]]
-;;                  (read-tuples results "2")))
+;;       (let [topology (topology
+;;                       {1 (spout-spec ConstantSpout)}
+;;                       {2 (bolt-spec {1 :shuffle} local-error-checker)
+;;                        })]
+;;         (submit-local-topology (:nimbus cluster)
+;;                                "test"
+;;                                {}
+;;                                topology)
+;;         (while (not @restarted)
+;;           (advance-time-ms! 100))
 ;;         )))
-;;
-;; (deftest test-shuffle
-;;   (with-simulated-time-local-cluster [cluster :supervisors 4]
-;;     (let [topology (thrift/mk-topology
-;;                     {"1" (thrift/mk-spout-spec (TestWordSpout. true) :parallelism-hint 4)}
-;;                     {"2" (thrift/mk-bolt-spec {"1" :shuffle} (TestGlobalCount.)
-;;                                             :parallelism-hint 6)
-;;                      })
-;;           results (complete-topology cluster
-;;                                      topology
-;;                                      ;; important for test that
-;;                                      ;; #tuples = multiple of 4 and 6
-;;                                      :mock-sources {"1" [["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ["a"] ["b"]
-;;                                                        ]}
-;;                                      )]
-;;       (is (ms= (apply concat (repeat 6 [[1] [2] [3] [4]]))
-;;                (read-tuples results "2")))
+
+(defspout IncSpout ["word"]
+  [conf context collector]
+  (let [state (atom 0)]
+    (spout
+     (nextTuple []
+       (Thread/sleep 100)
+       (emit-spout! collector [@state] :id 1)
+       )
+     (ack [id]
+       (swap! state inc))
+     )))
+
+
+(defspout IncSpout2 ["word"] {:params [prefix]}
+  [conf context collector]
+  (let [state (atom 0)]
+    (spout
+     (nextTuple []
+       (Thread/sleep 100)
+       (swap! state inc)
+       (emit-spout! collector [(str prefix "-" @state)])
+       )
+     )))
+
+;; (deftest test-clojure-spout
+;;   (with-local-cluster [cluster]
+;;     (let [nimbus (:nimbus cluster)
+;;           top (topology
+;;                {1 (spout-spec IncSpout)}
+;;                {}
+;;                )]
+;;       (submit-local-topology nimbus
+;;                              "spout-test"
+;;                              {TOPOLOGY-DEBUG true
+;;                               TOPOLOGY-MESSAGE-TIMEOUT-SECS 3}
+;;                              top)
+;;       (Thread/sleep 10000)
+;;       (.killTopology nimbus "spout-test")
+;;       (Thread/sleep 10000)
 ;;       )))
-;;
-;; (defbolt lalala-bolt1 ["word"] [[val :as tuple] collector]
-;;   (let [ret (str val "lalala")]
-;;     (emit-bolt! collector [ret] :anchor tuple)
-;;     (ack! collector tuple)
-;;     ))
-;;
-;; (defbolt lalala-bolt2 ["word"] {:prepare true}
-;;   [conf context collector]
-;;   (let [state (atom nil)]
-;;     (reset! state "lalala")
-;;     (bolt
-;;       (execute [tuple]
-;;         (let [ret (-> (.getValue tuple 0) (str @state))]
-;;                 (emit-bolt! collector [ret] :anchor tuple)
-;;                 (ack! collector tuple)
-;;                 ))
-;;       )))
-;;
-;; (defbolt lalala-bolt3 ["word"] {:prepare true :params [prefix]}
-;;   [conf context collector]
-;;   (let [state (atom nil)]
-;;     (bolt
-;;       (prepare [_ _ _]
-;;         (reset! state (str prefix "lalala")))
-;;       (execute [{val "word" :as tuple}]
-;;         (let [ret (-> (.getValue tuple 0) (str @state))]
-;;           (emit-bolt! collector [ret] :anchor tuple)
-;;           (ack! collector tuple)
-;;           )))
-;;     ))
-;;
-;; (deftest test-clojure-bolt
-;;   (with-simulated-time-local-cluster [cluster :supervisors 4]
-;;     (let [nimbus (:nimbus cluster)
-;;           topology (thrift/mk-topology
-;;                       {"1" (thrift/mk-spout-spec (TestWordSpout. false))}
-;;                       {"2" (thrift/mk-bolt-spec {"1" :shuffle}
-;;                                               lalala-bolt1)
-;;                        "3" (thrift/mk-bolt-spec {"1" :shuffle}
-;;                                               lalala-bolt2)
-;;                        "4" (thrift/mk-bolt-spec {"1" :shuffle}
-;;                                               (lalala-bolt3 "_nathan_"))}
-;;                       )
-;;           results (complete-topology cluster
-;;                                      topology
-;;                                      :mock-sources {"1" [["david"]
-;;                                                        ["adam"]
-;;                                                        ]}
-;;                                      )]
-;;       (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "2")))
-;;       (is (ms= [["davidlalala"] ["adamlalala"]] (read-tuples results "3")))
-;;       (is (ms= [["david_nathan_lalala"] ["adam_nathan_lalala"]] (read-tuples results "4")))
-;;       )))
-;;
-;; (defbolt punctuator-bolt ["word" "period" "question" "exclamation"]
-;;   [tuple collector]
-;;   (if (= (:word tuple) "bar")
-;;     (do
-;;       (emit-bolt! collector {:word "bar" :period "bar" :question "bar"
-;;                             "exclamation" "bar"})
-;;       (ack! collector tuple))
-;;     (let [ res (assoc tuple :period (str (:word tuple) "."))
-;;            res (assoc res :exclamation (str (:word tuple) "!"))
-;;            res (assoc res :question (str (:word tuple) "?")) ]
-;;       (emit-bolt! collector res)
-;;       (ack! collector tuple))))
-;;
-;; (deftest test-map-emit
-;;   (with-simulated-time-local-cluster [cluster :supervisors 4]
-;;     (let [topology (thrift/mk-topology
-;;                       {"words" (thrift/mk-spout-spec (TestWordSpout. false))}
-;;                       {"out" (thrift/mk-bolt-spec {"words" :shuffle}
-;;                                               punctuator-bolt)}
-;;                       )
-;;           results (complete-topology cluster
-;;                                      topology
-;;                                      :mock-sources {"words" [["foo"] ["bar"]]}
-;;                                      )]
-;;       (is (ms= [["foo" "foo." "foo?" "foo!"]
-;;                 ["bar" "bar" "bar" "bar"]   ] (read-tuples results "out"))))))
-;;
-;;
-;; (defn ack-tracking-feeder [fields]
-;;   (let [tracker (AckTracker.)]
-;;     [(doto (feeder-spout fields)
-;;        (.setAckFailDelegate tracker))
-;;      (fn [val]
-;;        (is (= (.getNumAcks tracker) val))
-;;        (.resetNumAcks tracker)
-;;        )]
-;;     ))
-;;
-;; (defbolt branching-bolt ["num"]
-;;   {:params [amt]}
-;;   [tuple collector]
-;;   (doseq [i (range amt)]
-;;     (emit-bolt! collector [i] :anchor tuple))
-;;   (ack! collector tuple))
-;;
-;; (defbolt agg-bolt ["num"] {:prepare true :params [amt]}
-;;   [conf context collector]
-;;   (let [seen (atom [])]
-;;     (bolt
-;;       (execute [tuple]
-;;         (swap! seen conj tuple)
-;;         (when (= (count @seen) amt)
-;;           (emit-bolt! collector [1] :anchor @seen)
-;;           (doseq [s @seen]
-;;             (ack! collector s))
-;;           (reset! seen [])
-;;           )))
-;;       ))
-;;
-;; (defbolt ack-bolt {}
-;;   [tuple collector]
-;;   (ack! collector tuple))
-;;
-;; (deftest test-acking
-;;   (with-tracked-cluster [cluster]
-;;     (let [[feeder1 checker1] (ack-tracking-feeder ["num"])
-;;           [feeder2 checker2] (ack-tracking-feeder ["num"])
-;;           [feeder3 checker3] (ack-tracking-feeder ["num"])
-;;           tracked (mk-tracked-topology
-;;                    cluster
-;;                    (topology
-;;                      {"1" (spout-spec feeder1)
-;;                       "2" (spout-spec feeder2)
-;;                       "3" (spout-spec feeder3)}
-;;                      {"4" (bolt-spec {"1" :shuffle} (branching-bolt 2))
-;;                       "5" (bolt-spec {"2" :shuffle} (branching-bolt 4))
-;;                       "6" (bolt-spec {"3" :shuffle} (branching-bolt 1))
-;;                       "7" (bolt-spec
-;;                             {"4" :shuffle
-;;                             "5" :shuffle
-;;                             "6" :shuffle}
-;;                             (agg-bolt 3))
-;;                       "8" (bolt-spec {"7" :shuffle} (branching-bolt 2))
-;;                       "9" (bolt-spec {"8" :shuffle} ack-bolt)}
-;;                      ))]
-;;       (submit-local-topology (:nimbus cluster)
-;;                              "acking-test1"
-;;                              {TOPOLOGY-DEBUG true}
-;;                              (:topology tracked))
-;;       (.feed feeder1 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 0)
-;;       (.feed feeder2 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 1)
-;;       (checker2 1)
-;;       (.feed feeder1 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 0)
-;;       (.feed feeder1 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 1)
-;;       (.feed feeder3 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 0)
-;;       (checker3 0)
-;;       (.feed feeder2 [1])
-;;       (tracked-wait tracked 1)
-;;       (checker1 1)
-;;       (checker2 1)
-;;       (checker3 1)
-;;
-;;       )))
-;;
-;; (deftest test-ack-branching
-;;   (with-tracked-cluster [cluster]
-;;     (let [[feeder checker] (ack-tracking-feeder ["num"])
-;;           tracked (mk-tracked-topology
-;;                    cluster
-;;                    (topology
-;;                      {"1" (spout-spec feeder)}
-;;                      {"2" (bolt-spec {"1" :shuffle} identity-bolt)
-;;                       "3" (bolt-spec {"1" :shuffle} identity-bolt)
-;;                       "4" (bolt-spec
-;;                             {"2" :shuffle
-;;                              "3" :shuffle}
-;;                              (agg-bolt 4))}))]
-;;       (submit-local-topology (:nimbus cluster)
-;;                              "test-acking2"
-;;                              {}
-;;                              (:topology tracked))
-;;       (.feed feeder [1])
-;;       (tracked-wait tracked 1)
-;;       (checker 0)
-;;       (.feed feeder [1])
-;;       (tracked-wait tracked 1)
-;;       (checker 2)
-;;       )))
-;;
-;; (defbolt dup-anchor ["num"]
-;;   [tuple collector]
-;;   (emit-bolt! collector [1] :anchor [tuple tuple])
-;;   (ack! collector tuple))
-;;
-;; (deftest test-acking-self-anchor
-;;   (with-tracked-cluster [cluster]
-;;     (let [[feeder checker] (ack-tracking-feeder ["num"])
-;;           tracked (mk-tracked-topology
-;;                    cluster
-;;                    (topology
-;;                      {"1" (spout-spec feeder)}
-;;                      {"2" (bolt-spec {"1" :shuffle} dup-anchor)
-;;                       "3" (bolt-spec {"2" :shuffle} ack-bolt)}))]
-;;       (submit-local-topology (:nimbus cluster)
-;;                              "test"
-;;                              {}
-;;                              (:topology tracked))
-;;       (.feed feeder [1])
-;;       (tracked-wait tracked 1)
-;;       (checker 1)
-;;       (.feed feeder [1])
-;;       (.feed feeder [1])
-;;       (.feed feeder [1])
-;;       (tracked-wait tracked 3)
-;;       (checker 3)
-;;       )))
-;;
-;; ;; (defspout ConstantSpout ["val"] {:prepare false}
-;; ;;   [collector]
-;; ;;   (Time/sleep 100)
-;; ;;   (emit-spout! collector [1]))
-;;
-;; ;; (def errored (atom false))
-;; ;; (def restarted (atom false))
-;;
-;; ;; (defbolt local-error-checker {} [tuple collector]
-;; ;;   (when-not @errored
-;; ;;     (reset! errored true)
-;; ;;     (println "erroring")
-;; ;;     (throw (RuntimeException.)))
-;; ;;   (when-not @restarted (println "restarted"))
-;; ;;   (reset! restarted true))
-;;
-;; ;; (deftest test-no-halt-local-mode
-;; ;;   (with-simulated-time-local-cluster [cluster]
-;; ;;       (let [topology (topology
-;; ;;                       {1 (spout-spec ConstantSpout)}
-;; ;;                       {2 (bolt-spec {1 :shuffle} local-error-checker)
-;; ;;                        })]
-;; ;;         (submit-local-topology (:nimbus cluster)
-;; ;;                                "test"
-;; ;;                                {}
-;; ;;                                topology)
-;; ;;         (while (not @restarted)
-;; ;;           (advance-time-ms! 100))
-;; ;;         )))
-;;
-;; (defspout IncSpout ["word"]
-;;   [conf context collector]
-;;   (let [state (atom 0)]
-;;     (spout
-;;      (nextTuple []
-;;        (Thread/sleep 100)
-;;        (emit-spout! collector [@state] :id 1)
-;;        )
-;;      (ack [id]
-;;        (swap! state inc))
-;;      )))
-;;
-;;
-;; (defspout IncSpout2 ["word"] {:params [prefix]}
-;;   [conf context collector]
-;;   (let [state (atom 0)]
-;;     (spout
-;;      (nextTuple []
-;;        (Thread/sleep 100)
-;;        (swap! state inc)
-;;        (emit-spout! collector [(str prefix "-" @state)])
-;;        )
-;;      )))
-;;
-;; ;; (deftest test-clojure-spout
-;; ;;   (with-local-cluster [cluster]
-;; ;;     (let [nimbus (:nimbus cluster)
-;; ;;           top (topology
-;; ;;                {1 (spout-spec IncSpout)}
-;; ;;                {}
-;; ;;                )]
-;; ;;       (submit-local-topology nimbus
-;; ;;                              "spout-test"
-;; ;;                              {TOPOLOGY-DEBUG true
-;; ;;                               TOPOLOGY-MESSAGE-TIMEOUT-SECS 3}
-;; ;;                              top)
-;; ;;       (Thread/sleep 10000)
-;; ;;       (.killTopology nimbus "spout-test")
-;; ;;       (Thread/sleep 10000)
-;; ;;       )))
-;;
-;;
-;; (deftest test-component-specific-config
-;;   (with-simulated-time-local-cluster [cluster
-;;                                       :daemon-conf {TOPOLOGY-OPTIMIZE false
-;;                                                     TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true}]
-;;     (letlocals
-;;      (bind builder (TopologyBuilder.))
-;;      (.setSpout builder "1" (TestPlannerSpout. (Fields. ["conf"])))
-;;      (-> builder
-;;          (.setBolt "2"
-;;                    (TestConfBolt.
-;;                     {"fake.config" 123
-;;                      TOPOLOGY-MAX-TASK-PARALLELISM 20
-;;                      TOPOLOGY-MAX-SPOUT-PENDING 30
-;;                      TOPOLOGY-OPTIMIZE true
-;;                      TOPOLOGY-KRYO-REGISTER [{"fake.type" "bad.serializer"}
-;;                                              {"fake.type2" "a.serializer"}]
-;;                      }))
-;;          (.shuffleGrouping "1")
-;;          (.setMaxTaskParallelism 2)
-;;          (.addConfiguration "fake.config2" 987)
-;;          )
-;;
-;;
-;;      (bind results
-;;            (complete-topology cluster
-;;                               (.createTopology builder)
-;;                               :storm-conf {TOPOLOGY-KRYO-REGISTER [{"fake.type" "good.serializer" "fake.type3" "a.serializer3"}]}
-;;                               :mock-sources {"1" [["fake.config"]
-;;                                                   [TOPOLOGY-MAX-TASK-PARALLELISM]
-;;                                                   [TOPOLOGY-MAX-SPOUT-PENDING]
-;;                                                   [TOPOLOGY-OPTIMIZE]
-;;                                                   ["fake.config2"]
-;;                                                   [TOPOLOGY-KRYO-REGISTER]
-;;                                                   ]}))
-;;      (is (= {"fake.config" 123
-;;              "fake.config2" 987
-;;              TOPOLOGY-MAX-TASK-PARALLELISM 2
-;;              TOPOLOGY-MAX-SPOUT-PENDING 30
-;;              TOPOLOGY-OPTIMIZE false
-;;              TOPOLOGY-KRYO-REGISTER {"fake.type" "good.serializer"
-;;                                      "fake.type2" "a.serializer"
-;;                                      "fake.type3" "a.serializer3"}}
-;;             (->> (read-tuples results "2")
-;;                  (apply concat)
-;;                  (apply hash-map))
-;;             ))
-;;      )))
-;;
-;; (defbolt conf-query-bolt ["conf" "val"] {:prepare true :params [conf] :conf conf}
-;;   [conf context collector]
-;;   (bolt
-;;    (execute [tuple]
-;;             (let [name (.getValue tuple 0)
-;;                   val (if (= name "!MAX_MSG_TIMEOUT") (.maxTopologyMessageTimeout context) (get conf name))]
-;;               (emit-bolt! collector [name val] :anchor tuple)
-;;               (ack! collector tuple))
-;;             )))
-;;
-;; (deftest test-component-specific-config-clojure
-;;   (with-simulated-time-local-cluster [cluster]
-;;     (let [topology (topology {"1" (spout-spec (TestPlannerSpout. (Fields. ["conf"])) :conf {TOPOLOGY-MESSAGE-TIMEOUT-SECS 40})
-;;                               }
-;;                              {"2" (bolt-spec {"1" :shuffle}
-;;                                              (conf-query-bolt {"fake.config" 1
-;;                                                                TOPOLOGY-MAX-TASK-PARALLELISM 2
-;;                                                                TOPOLOGY-MAX-SPOUT-PENDING 10})
-;;                                              :conf {TOPOLOGY-MAX-SPOUT-PENDING 3})
-;;                               })
-;;           results (complete-topology cluster
-;;                                      topology
-;;                                      :topology-name "test123"
-;;                                      :storm-conf {TOPOLOGY-MAX-TASK-PARALLELISM 10
-;;                                                   TOPOLOGY-MESSAGE-TIMEOUT-SECS 30}
-;;                                      :mock-sources {"1" [["fake.config"]
-;;                                                          [TOPOLOGY-MAX-TASK-PARALLELISM]
-;;                                                          [TOPOLOGY-MAX-SPOUT-PENDING]
-;;                                                          ["!MAX_MSG_TIMEOUT"]
-;;                                                          [TOPOLOGY-NAME]
-;;                                                          ]})]
-;;       (is (= {"fake.config" 1
-;;               TOPOLOGY-MAX-TASK-PARALLELISM 2
-;;               TOPOLOGY-MAX-SPOUT-PENDING 3
-;;               "!MAX_MSG_TIMEOUT" 40
-;;               TOPOLOGY-NAME "test123"}
-;;              (->> (read-tuples results "2")
-;;                   (apply concat)
-;;                   (apply hash-map))
-;;              )))))
-;;
-;; (defbolt hooks-bolt ["emit" "ack" "fail"] {:prepare true}
-;;   [conf context collector]
-;;   (let [acked (atom 0)
-;;         failed (atom 0)
-;;         emitted (atom 0)]
-;;     (.addTaskHook context
-;;                   (reify backtype.storm.hooks.ITaskHook
-;;                     (prepare [this conf context]
-;;                       )
-;;                     (cleanup [this]
-;;                       )
-;;                     (emit [this info]
-;;                       (swap! emitted inc))
-;;                     (boltAck [this info]
-;;                       (swap! acked inc))
-;;                     (boltFail [this info]
-;;                       (swap! failed inc))))
-;;     (bolt
-;;      (execute [tuple]
-;;         (emit-bolt! collector [@emitted @acked @failed])
-;;         (if (= 0 (- @acked @failed))
-;;           (ack! collector tuple)
-;;           (fail! collector tuple))
-;;         ))))
-;;
-;; (deftest test-hooks
-;;   (with-simulated-time-local-cluster [cluster]
-;;     (let [topology (topology {"1" (spout-spec (TestPlannerSpout. (Fields. ["conf"])))
-;;                               }
-;;                              {"2" (bolt-spec {"1" :shuffle}
-;;                                              hooks-bolt)
-;;                               })
-;;           results (complete-topology cluster
-;;                                      topology
-;;                                      :mock-sources {"1" [[1]
-;;                                                          [1]
-;;                                                          [1]
-;;                                                          [1]
-;;                                                          ]})]
-;;       (is (= [[0 0 0]
-;;               [2 1 0]
-;;               [4 1 1]
-;;               [6 2 1]]
-;;              (read-tuples results "2")
-;;              )))))
-;;
-;; (deftest test-acking-branching-complex
-;;   ;; test acking with branching in the topology
-;;   )
-;;
-;;
-;; (deftest test-fields-grouping
-;;   ;; 1. put a shitload of random tuples through it and test that counts are right
-;;   ;; 2. test that different spouts with different phints group the same way
-;;   )
-;;
-;; (deftest test-all-grouping
-;;   )
-;;
-;; (deftest test-direct-grouping
-;;   )
+
+
+(deftest test-component-specific-config
+  (with-simulated-time-local-cluster [cluster
+                                      :daemon-conf {TOPOLOGY-OPTIMIZE false
+                                                    TOPOLOGY-SKIP-MISSING-KRYO-REGISTRATIONS true}]
+    (letlocals
+     (bind builder (TopologyBuilder.))
+     (.setSpout builder "1" (TestPlannerSpout. (Fields. ["conf"])))
+     (-> builder
+         (.setBolt "2"
+                   (TestConfBolt.
+                    {"fake.config" 123
+                     TOPOLOGY-MAX-TASK-PARALLELISM 20
+                     TOPOLOGY-MAX-SPOUT-PENDING 30
+                     TOPOLOGY-OPTIMIZE true
+                     TOPOLOGY-KRYO-REGISTER [{"fake.type" "bad.serializer"}
+                                             {"fake.type2" "a.serializer"}]
+                     }))
+         (.shuffleGrouping "1")
+         (.setMaxTaskParallelism 2)
+         (.addConfiguration "fake.config2" 987)
+         )
+
+
+     (bind results
+           (complete-topology cluster
+                              (.createTopology builder)
+                              :storm-conf {TOPOLOGY-KRYO-REGISTER [{"fake.type" "good.serializer" "fake.type3" "a.serializer3"}]}
+                              :mock-sources {"1" [["fake.config"]
+                                                  [TOPOLOGY-MAX-TASK-PARALLELISM]
+                                                  [TOPOLOGY-MAX-SPOUT-PENDING]
+                                                  [TOPOLOGY-OPTIMIZE]
+                                                  ["fake.config2"]
+                                                  [TOPOLOGY-KRYO-REGISTER]
+                                                  ]}))
+     (is (= {"fake.config" 123
+             "fake.config2" 987
+             TOPOLOGY-MAX-TASK-PARALLELISM 2
+             TOPOLOGY-MAX-SPOUT-PENDING 30
+             TOPOLOGY-OPTIMIZE false
+             TOPOLOGY-KRYO-REGISTER {"fake.type" "good.serializer"
+                                     "fake.type2" "a.serializer"
+                                     "fake.type3" "a.serializer3"}}
+            (->> (read-tuples results "2")
+                 (apply concat)
+                 (apply hash-map))
+            ))
+     )))
+
+(defbolt conf-query-bolt ["conf" "val"] {:prepare true :params [conf] :conf conf}
+  [conf context collector]
+  (bolt
+   (execute [tuple]
+            (let [name (.getValue tuple 0)
+                  val (if (= name "!MAX_MSG_TIMEOUT") (.maxTopologyMessageTimeout context) (get conf name))]
+              (emit-bolt! collector [name val] :anchor tuple)
+              (ack! collector tuple))
+            )))
+
+(deftest test-component-specific-config-clojure
+  (with-simulated-time-local-cluster [cluster]
+    (let [topology (topology {"1" (spout-spec (TestPlannerSpout. (Fields. ["conf"])) :conf {TOPOLOGY-MESSAGE-TIMEOUT-SECS 40})
+                              }
+                             {"2" (bolt-spec {"1" :shuffle}
+                                             (conf-query-bolt {"fake.config" 1
+                                                               TOPOLOGY-MAX-TASK-PARALLELISM 2
+                                                               TOPOLOGY-MAX-SPOUT-PENDING 10})
+                                             :conf {TOPOLOGY-MAX-SPOUT-PENDING 3})
+                              })
+          results (complete-topology cluster
+                                     topology
+                                     :topology-name "test123"
+                                     :storm-conf {TOPOLOGY-MAX-TASK-PARALLELISM 10
+                                                  TOPOLOGY-MESSAGE-TIMEOUT-SECS 30}
+                                     :mock-sources {"1" [["fake.config"]
+                                                         [TOPOLOGY-MAX-TASK-PARALLELISM]
+                                                         [TOPOLOGY-MAX-SPOUT-PENDING]
+                                                         ["!MAX_MSG_TIMEOUT"]
+                                                         [TOPOLOGY-NAME]
+                                                         ]})]
+      (is (= {"fake.config" 1
+              TOPOLOGY-MAX-TASK-PARALLELISM 2
+              TOPOLOGY-MAX-SPOUT-PENDING 3
+              "!MAX_MSG_TIMEOUT" 40
+              TOPOLOGY-NAME "test123"}
+             (->> (read-tuples results "2")
+                  (apply concat)
+                  (apply hash-map))
+             )))))
+
+(defbolt hooks-bolt ["emit" "ack" "fail"] {:prepare true}
+  [conf context collector]
+  (let [acked (atom 0)
+        failed (atom 0)
+        emitted (atom 0)]
+    (.addTaskHook context
+                  (reify backtype.storm.hooks.ITaskHook
+                    (prepare [this conf context]
+                      )
+                    (cleanup [this]
+                      )
+                    (emit [this info]
+                      (swap! emitted inc))
+                    (boltAck [this info]
+                      (swap! acked inc))
+                    (boltFail [this info]
+                      (swap! failed inc))))
+    (bolt
+     (execute [tuple]
+        (emit-bolt! collector [@emitted @acked @failed])
+        (if (= 0 (- @acked @failed))
+          (ack! collector tuple)
+          (fail! collector tuple))
+        ))))
+
+(deftest test-hooks
+  (with-simulated-time-local-cluster [cluster]
+    (let [topology (topology {"1" (spout-spec (TestPlannerSpout. (Fields. ["conf"])))
+                              }
+                             {"2" (bolt-spec {"1" :shuffle}
+                                             hooks-bolt)
+                              })
+          results (complete-topology cluster
+                                     topology
+                                     :mock-sources {"1" [[1]
+                                                         [1]
+                                                         [1]
+                                                         [1]
+                                                         ]})]
+      (is (= [[0 0 0]
+              [2 1 0]
+              [4 1 1]
+              [6 2 1]]
+             (read-tuples results "2")
+             )))))
+
+(deftest test-acking-branching-complex
+  ;; test acking with branching in the topology
+  )
+
+
+(deftest test-fields-grouping
+  ;; 1. put a shitload of random tuples through it and test that counts are right
+  ;; 2. test that different spouts with different phints group the same way
+  )
+
+(deftest test-all-grouping
+  )
+
+(deftest test-direct-grouping
+  )


"a OS X plan" => "an OS X plan"

"a OS X plan" => "an OS X plan"

Havin' a grammar opinion about indefinite articles, even though I know this is about to be displaced by macOS.

--- src-cljs/frontend/components/org_settings.cljs
+++ src-cljs/frontend/components/org_settings.cljs
@@ -1319,7 +1319,7 @@
                  (om/build osx-usage-table-plan {:plan plan})]])
              [:div.plan-header
               [:h1 "No OS X plan selected"]
-              [:p "Choose a OS X plan below."]])])))))
+              [:p "Choose an OS X plan below."]])])))))

 (defn osx-usage-table [{:keys [plan]} owner]
   (reify
@@ -1437,7 +1437,7 @@
              [:p
               [:strong [:a {:href (routes/v1-org-settings-path {:org plan-org-name
                                                                 :vcs_type plan-vcs-type
-                                                                :_fragment "osx-pricing"})} "Choose a OS X plan"]] "."]])])))))
+                                                                :_fragment "osx-pricing"})} "Choose an OS X plan"]] "."]])])))))

 (defn displayed-linux-plan-info [current-linux-cost containers in-student-trial?]
   (gstring/format "Current Linux plan: %s - $%s/month"


Make sure that the number of regions is printed as integer

--- src/main/hbase_region_inspector/core.clj
+++ src/main/hbase_region_inspector/core.clj
@@ -47,6 +47,7 @@
     (case type
       :start-key                ["Start key"  (hbase/byte-array->str val)]
       :end-key                  ["End key"    (hbase/byte-array->str val)]
+      :regions                  ["Regions"    (long-fmt val)]
       :store-files              ["Storefiles" (long-fmt val)]
       :store-file-size-mb       ["Data size"
                                  (if-let [uncmp (:store-uncompressed-size-mb props)]


fix compilation hang

fix compilation hang

--- dada-swt/src/main/clojure/org/dada/swt/new.clj
+++ dada-swt/src/main/clojure/org/dada/swt/new.clj
@@ -23,13 +23,14 @@

 ;;--------------------------------------------------------------------------------

-(if (not *compile-files*)
-  (def ^Display *display* (Display.))
-  (.start
-   (Thread.
-    (fn []
-	(def ^Display *display* (Display.))
-	(swt-loop *display*)))))
+ (if (not *compile-files*)
+   (do
+     (def ^Display *display* (Display.))
+     (.start
+      (Thread.
+       (fn []
+	   (def ^Display *display* (Display.))
+	   (swt-loop *display*))))))

 ;; TODO - detach View on closing
 (defn inspect [query]


backpressure off was not being set, due to code being overwritten

--- src/onyx/log/commands/backpressure_off.clj
+++ src/onyx/log/commands/backpressure_off.clj
@@ -9,7 +9,7 @@
 (defmethod extensions/apply-log-entry :backpressure-off
   [{:keys [args]} replica]
   (if (= :backpressure (get-in replica [:peer-state (:peer args)]))
-    (assoc-in replica [:peer-state (:peer args)] :backpressure)
+    (assoc-in replica [:peer-state (:peer args)] :active)
     replica)

   (assoc-in replica [:peer-state (:peer args)] :active))


make 60 the baseline for entity ids

make 60 the baseline for entity ids

--- src-cljs/frontend/core.cljs
+++ src-cljs/frontend/core.cljs
@@ -291,7 +291,7 @@

 (defn setup-entity-id-fetcher [state]
   (let [api-ch (-> state deref :comms :api)]
-    (fetch-entity-ids api-ch 40)
+    (fetch-entity-ids api-ch 60)
     (add-watch state :entity-id-fetcher (fn [key ref old new]
                                           (when (> 20 (-> new :entity-ids count))
                                             (println "fetching more entity ids")


Made scale easier to understand.

Made scale easier to understand.

--- src/overtunes/songs/canone_alla_quarta.clj
+++ src/overtunes/songs/canone_alla_quarta.clj
@@ -7,10 +7,9 @@
 (defn sum-n [series n] (reduce + (take n series)))
 (defn sums [series] (cons 0 (reductions + series)))
 (defn scale [intervals]
-  #(if (neg? %)
-     (let [downward-scale (comp - (scale (reverse intervals)))]
-       (-> % - downward-scale))
-     (sum-n (cycle intervals) %)))
+  #(if (not (neg? %))
+     (sum-n (cycle intervals) %)
+     (=> % - (scale (reverse intervals)) -)))

 (def t 0)
 (def p 1)


[#2309] Uncomment wrap-jwt-authentication init-key

--- backend/src/akvo/lumen/auth.clj
+++ backend/src/akvo/lumen/auth.clj
@@ -16,13 +16,13 @@
 (s/def ::auth0-public-client ::auth0/public-client)

-;; (defmethod ig/init-key :akvo.lumen.auth/wrap-jwt-authentication
-;;   [_ {:keys [keycloak-public-client auth0-public-client authorizer]}]
-;;   (wrap-jwt-authentication keycloak-public-client auth0-public-client authorizer))
+(defmethod ig/init-key :akvo.lumen.auth/wrap-jwt-authentication
+  [_ {:keys [keycloak-public-client auth0-public-client authorizer]}]
+  (wrap-jwt-authentication keycloak-public-client auth0-public-client authorizer))

-;; (defmethod ig/pre-init-spec :akvo.lumen.auth/wrap-jwt-authentication
-;;   [_]
-;;   (s/keys :req-un [::keycloak-public-client ::auth0-public-client ::p/authorizer]))
+(defmethod ig/pre-init-spec :akvo.lumen.auth/wrap-jwt-authentication
+  [_]
+  (s/keys :req-un [::keycloak-public-client ::auth0-public-client ::p/authorizer]))

 ;; (defmethod ig/pre-init-spec :akvo.lumen.auth/wrap-jwt-authorization
 ;;   [_]


forgot to document one route

forgot to document one route

--- src/restaurant_inspections_api/schemas.clj
+++ src/restaurant_inspections_api/schemas.clj
@@ -162,7 +162,7 @@
                                          :summary "List Inspections using filters"}}

                    "/inspections/:id" {:get {:responses {200 {:schema InspectionDetail
-                                                              :description "Found it!"}}
+                                                              :description "Individual Inspection Detail"}}
                                              :parameters {:path {:id Long}}
                                              :summary "Retrieve one inspection by id; includes violations details"}}


Add sampling option to search

Add sampling option to search

--- src/angelic/search/implicit/dash_astar_opt.clj
+++ src/angelic/search/implicit/dash_astar_opt.clj
@@ -464,6 +464,18 @@

 (declare *root*)

+(defn wtd-sample [item-wt-pairs]
+  (let [tot-wt (reduce + (map second item-wt-pairs))]
+    (loop [[[item wt] & more-items] (seq item-wt-pairs)]
+      (cond (empty? more-items) item
+            (<= (rand) (/ wt tot-wt)) item
+            :else (recur more-items)))))
+
+;; Doesn't really do what's intended, since solved wt is counted too.
+(defn even-sample [nodes]
+  (wtd-sample (for [n nodes] [n (inc (- (summary/max-reward (sg/summary n))))])))
+
+
 (defn implicit-dash-a*-opt
   "Solve this hierarchical env using implicit DASH-A*, or variants thereupon.  Options are:
     :gather    Don't publish child subproblems with identical outputs?


Move the info message to the proper place

--- src/org/spootnik/cyanite/store_cache.clj
+++ src/org/spootnik/cyanite/store_cache.clj
@@ -108,13 +108,13 @@

 (defn simple-cache
   [{:keys [store fn-agg] :or {fn-agg agg-avg}}]
+  (info "creating simple store aggregation cache")
   (let [mkeys (atom {})
         get-data (fn [pkeys] (get (meta pkeys) :data))
         cache-get (fn [key pkeys] (get @(get-data pkeys) key))
         cache-key (fn [tenant period rollup time path] (hash path))
         schan (store/channel-for store)
         fn-store (partial store-chan schan)]
-    (info "creating simple store aggregation cache")
     (reify
       StoreCache
       (put! [this tenant period rollup time path data ttl]


Cosmetics

Cosmetics

--- src/clojure/clojurewerkz/cassaforte/client.clj
+++ src/clojure/clojurewerkz/cassaforte/client.clj
@@ -290,9 +290,9 @@
   "Exports the schema as a string"
   [^Session client]
   (-> client
-      (.getCluster)
-      (.getMetadata)
-      (.exportSchemaAsString)))
+      .getCluster
+      .getMetadata
+      .exportSchemaAsString))

 (defn get-hosts
   "Returns all nodes in the cluster"
@@ -303,9 +303,9 @@
           :rack       (.getRack host)
           :is-up      (.isUp (.getMonitor host))})
        (-> session
-           (.getCluster)
-           (.getMetadata)
-           (.getAllHosts))))
+           .getCluster
+           .getMetadata
+           .getAllHosts)))

 ;; defn get-replicas
 ;; defn get-cluster-name
@@ -314,11 +314,11 @@
 ;; defn rebuild-schema

 ;;
-;;
+;; Result Handling
 ;;

 (defn set-callbacks
-  "Set callbacks for the future"
+  "Set callbacks on a result future"
   [^ResultSetFuture future & {:keys [success failure]}]
   {:pre [(not (nil? success))]}
   (Futures/addCallback


Show upload percentage

Show upload percentage

--- src/airhead_cljs/requests.cljs
+++ src/airhead_cljs/requests.cljs
@@ -1,8 +1,10 @@
 (ns airhead-cljs.requests
-  (:require-macros [cljs.core.async.macros :refer [go]])
+  (:require-macros [cljs.core.async.macros :refer [go go-loop]])
   (:require [cljs-http.client :as http]
-            [cljs.core.async :refer [<!]]
-            [airhead-cljs.state :refer [app-state update-state!]]))
+            [cljs.core.async :refer [<! chan]]
+            [airhead-cljs.state :refer [app-state update-state!]]
+            [goog.string :as gstring]
+            [goog.string.format]))

 ;; TODO: handle error responses

@@ -28,15 +30,25 @@
         (update-state! :library (get-in response [:body :tracks])))))

 (defn upload! []
- ;; TODO: do not use element id
- (let [form (.getElementById js/document
-             "upload-form")]
-  (update-state! :upload-status "Uploading...")
-  ;; TODO: progress bar
-  (go (let [response (<! (http/post "/api/library"
-                          {:body (js/FormData. form)}))]
-       (update-state! :upload-status
-        "Done! Now wait for the track being transcoded.")))))
+  ;; TODO: do not use element id
+  (let [form (.getElementById js/document "upload-form")
+        progress-chan (chan)
+        http-chan (http/post "/api/library" {:body (js/FormData. form)
+                                             :progress progress-chan})]
+    (go-loop []
+             (when-let [data (<! progress-chan)]
+               (when (= (data :direction) :upload)
+                 (let [loaded     (data :loaded)
+                       total      (data :total)
+                       percentage (-> loaded (/ total) (* 100))
+                       status     (gstring/format "Uploading: %.0f%"
+                                                  percentage)]
+                   (update-state! :upload-status status)))
+               (recur)))
+    (go (let [response (<! http-chan)]
+          ;; TODO: check response, and take transcoding status from a websocket
+          (update-state! :upload-status
+                         "Done! Now wait for the track being transcoded.")))))

 (defn polling-callback []
   (get-info!)


RR: add test namespaces to test runner

RR: add test namespaces to test runner

--- test/test_runner.cljs
+++ test/test_runner.cljs
@@ -2,6 +2,14 @@
   (:require
    [cljs.test :as test :refer-macros [run-tests] :refer [report]]
    [hatti.test-utils]
+   [hatti.map.utils-test]
+   [hatti.map.viewby-test]
+   [hatti.ona.post-process-test]
+   [hatti.views.chart-test]
+   [hatti.views.dataview-test]
+   [hatti.views.map-test]
+   [hatti.views.record-test]
+   [hatti.views.table-test]
    [hatti.shared-test]))

 (enable-console-print!)
@@ -16,6 +24,15 @@
   (if (cljs.test/successful?
        (run-tests
         (test/empty-env ::test/default)
+        'hatti.test-utils
+        'hatti.map.utils-test
+        'hatti.map.viewby-test
+        'hatti.ona.post-process-test
+        'hatti.views.chart-test
+        'hatti.views.dataview-test
+        'hatti.views.map-test
+        'hatti.views.record-test
+        'hatti.views.table-test
         'hatti.shared-test))
    0
    1))


Updating version to 1.62.1-SNAPSHOT

Updating version to 1.62.1-SNAPSHOT

--- scheduler/project.clj
+++ scheduler/project.clj
@@ -13,7 +13,7 @@
 ;; See the License for the specific language governing permissions and
 ;; limitations under the License.
 ;;
-(defproject cook "1.62.0"
+(defproject cook "1.62.1-SNAPSHOT"
   :description "This launches jobs on a Mesos cluster with fair sharing and preemption"
   :license {:name "Apache License, Version 2.0"}
   :dependencies [[org.clojure/clojure "1.10.3"]


+chat (intransitive)

+chat (intransitive)

--- src/babel/english/lexicon.cljc
+++ src/babel/english/lexicon.cljc
@@ -451,12 +451,14 @@
                                  :sem {:pred :chat}}
                         :english {:participle "chatting"
                                   :past "chatted"}}]
-            [common ;; intransitive
+            [(unify common ;; intransitive
+                    {:synsem {:subcat {:2 '()}}})
              (unify common ;; transitive "chatted with <human>"
                     {:synsem {:subcat {:2 {:cat :prep
                                            :sem {:obj {:human true}
                                                  :pred :with}
                                            :subcat '()}}}})])
+
    "charge" {:synsem {:cat :verb
                       :sem {:pred :caricare}}}


whitespace

whitespace

--- src/babel/english/lexicon.cljc
+++ src/babel/english/lexicon.cljc
@@ -1218,21 +1218,21 @@
   "Luisa and I"
    [{:english {:note "♂"}
      :synsem {:cat :noun
-             :pronoun true
-             :case :nom
-             :agr {:gender :masc
-                   :person :1st
-                   :number :plur}
-             :sem {:human true
-                   :pred :luisa-and-i}
+              :pronoun true
+              :case :nom
+              :agr {:gender :masc
+                    :person :1st
+                    :number :plur}
+              :sem {:human true
+                    :pred :luisa-and-i}
              :subcat '()}}
     {:english {:note "♀"}
      :synsem {:cat :noun
               :pronoun true
               :case :nom
-             :agr {:gender :fem
-                   :person :1st
-                   :number :plur}
+              :agr {:gender :fem
+                    :person :1st
+                    :number :plur}
               :sem {:human true
                     :pred :luisa-and-i}
               :subcat '()}}]


block intransitivization on shared-semantics verbs "e.g." be + PP

--- src/babel/english/lexicon.cljc
+++ src/babel/english/lexicon.cljc
@@ -2317,6 +2317,7 @@
                             :subcat {:1 {:top :top}
                                      :2 '()}
                             :sem {:reflexive false
+                                  :shared-with-obj false
                                   :obj :unspec}}})

                  ;; make an intransitive version of every verb which has an


add subcategorization for "help"

add subcategorization for "help"

--- src/babel/english/lexicon.cljc
+++ src/babel/english/lexicon.cljc
@@ -830,7 +830,11 @@
              :essere false
              :sem {:pred :aiutare
                    :activity true
-                   :obj {:human true}}}}
+                   :obj {:human true}}
+             :subcat {:1 {:cat :noun}
+                      :2 {:cat :noun
+                          :sem {:human true}}}}}
+
    "her"
    [{:synsem {:cat :det
               :agr {:number :sing}


"put on" is not a phrasal verb in our terminology (in contrast with "put X on" which *is* a phrasal verb)

--- src/babel/english/lexicon.cljc
+++ src/babel/english/lexicon.cljc
@@ -1696,7 +1696,6 @@
               :past "put on"
               :present {:3sing "puts on"}
               :note "past tense"}
-    :phrasal-verb true
     :intransitivize false
     :synsem {:cat :verb
              :sem {:pred :put-on}}}


Use 'node' as 'xml/element' in wpml package

--- src/bakyeono/litedocx/wpml.clj
+++ src/bakyeono/litedocx/wpml.clj
@@ -1,15 +1,21 @@
 (ns bakyeono.litedocx.wpml
   "WordProcessingML templates & snippets used in litedocx."
   (:require [clojure.string :as str])
-  (:use [clojure.data.xml :only [element]])
+  (:require [clojure.data.xml :as xml])
   (:use [bakyeono.litedocx.util]))

+;;; e -> clojure.data.xml/element
+;;; Since clojure.data.xml/element is used so much,
+;;; use 'node' as an abbreviation of it.
+;
+(def ^:const ^{:private true} node clojure.data.xml/element)
+
+;;; Constants
 (def ^:const ^{:private true} content-types-xml-xmlns
   {:xmlns "http://schemas.openxmlformats.org/package/2006/content-types"})

 (def ^:const ^{:private true} relationships-xmlns
   {:xmlns "http://schemas.openxmlformats.org/package/2006/relationships"})
-
 (def ^:const ^{:private true} doc-props-app-xml-xmlns
   {:xmlns:vt "http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes"
    :xmlns:properties "http://schemas.openxmlformats.org/officeDocument/2006/extended-properties"})
@@ -47,67 +53,66 @@
    :xmlns:w "http://schemas.openxmlformats.org/wordprocessingml/2006/main"})

 (def ^:const ^{:private true} default-content-types
-  [(element :Default
-            {:Extension "rels"
-             :ContentType "application/vnd.openxmlformats-package.relationships+xml"})
-   (element :Override
-            {:ContentType "application/vnd.openxmlformats-officedocument.extended-properties+xml"
-             :PartName "/docProps/app.xml"})
-   (element :Override
-            {:ContentType "application/vnd.openxmlformats-package.core-properties+xml"
-             :PartName "/docProps/core.xml"})
-   (element :Override
-            {:ContentType "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"
-             :PartName "/word/document.xml"})])
+  [(node :Default
+         {:Extension "rels"
+          :ContentType "application/vnd.openxmlformats-package.relationships+xml"})
+   (node :Override
+         {:ContentType "application/vnd.openxmlformats-officedocument.extended-properties+xml"
+          :PartName "/docProps/app.xml"})
+   (node :Override
+         {:ContentType "application/vnd.openxmlformats-package.core-properties+xml"
+          :PartName "/docProps/core.xml"})
+   (node :Override
+         {:ContentType "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"
+          :PartName "/word/document.xml"})])

 (def ^:const ^{:private true} default-styles
-  [(element :w:style
-            {:w:type "paragraph" :w:styleId "a" :w:default "true"}
-            [(element :w:name {:w:val "Normal"})
-             (element :w:qFormat)
-             (element :w:pPr
-                      {}
-                      [(element :w:widowControl {:w:val "false"})
-                       (element :w:wordWrap {:w:val "false"})
-                       (element :w:autoSpaceDE {:w:val "false"})
-                       (element :w:autoSpaceDN {:w:val "false"})
-                       (element :w:jc {:w:val "both"})])])
-   (element :w:style
-            {:w:type "character" :w:styleId "a0" :w:default "true"}
-            [(element :w:name {:w:val "Default Paragraph Font"})
-             (element :w:uiPriority {:w:val "1"})
-             (element :w:semiHidden)
-             (element :w:unhideWhenUsed)])
-   (element :w:style
-            {:w:type "table" :w:styleId "a1" :w:default "true"}
-            [(element :w:name {:w:val "Normal Table"})
-             (element :w:uiPriority {:w:val "99"})
-             (element :w:semiHidden)
-             (element :w:unhideWhenUsed)
-             (element :w:qFormat)
-             (element :w:tblPr
-                      {}
-                      [(element :w:tblInd {:w:w "0" :w:type "dxa"})
-                       (element :w:tblCellMar
-                                {}
-                                [(element :w:top {:w:w "0" :w:type "dxa"})
-                                 (element :w:left {:w:w "108" :w:type "dxa"})
-                                 (element :w:bottom {:w:w "0" :w:type "dxa"})
-                                 (element :w:right {:w:w "108" :w:type "dxa"})])])])
-   (element :w:style
-            {:w:type "numbering" :w:styleId "a2" :w:default "true"}
-            [(element :w:name {:w:val "No List"})
-             (element :w:uiPriority {:w:val "99"})
-             (element :w:semiHidden)
-             (element :w:unhideWhenUsed)])])
-
-(defn when-v-tag
+  [(node :w:style
+         {:w:type "paragraph" :w:styleId "a" :w:default "true"}
+         [(node :w:name {:w:val "Normal"})
+          (node :w:qFormat)
+          (node :w:pPr
+                {}
+                [(node :w:widowControl {:w:val "false"})
+                 (node :w:wordWrap {:w:val "false"})
+                 (node :w:autoSpaceDE {:w:val "false"})
+                 (node :w:autoSpaceDN {:w:val "false"})
+                 (node :w:jc {:w:val "both"})])])
+   (node :w:style
+         {:w:type "character" :w:styleId "a0" :w:default "true"}
+         [(node :w:name {:w:val "Default Paragraph Font"})
+          (node :w:uiPriority {:w:val "1"})
+          (node :w:semiHidden)
+          (node :w:unhideWhenUsed)])
+   (node :w:style
+         {:w:type "table" :w:styleId "a1" :w:default "true"}
+         [(node :w:name {:w:val "Normal Table"})
+          (node :w:uiPriority {:w:val "99"})
+          (node :w:semiHidden)
+          (node :w:unhideWhenUsed)
+          (node :w:qFormat)
+          (node :w:tblPr
+                {}
+                [(node :w:tblInd {:w:w "0" :w:type "dxa"})
+                 (node :w:tblCellMar
+                       {}
+                       [(node :w:top {:w:w "0" :w:type "dxa"})
+                        (node :w:left {:w:w "108" :w:type "dxa"})
+                        (node :w:bottom {:w:w "0" :w:type "dxa"})
+                        (node :w:right {:w:w "108" :w:type "dxa"})])])])
+   (node :w:style
+         {:w:type "numbering" :w:styleId "a2" :w:default "true"}
+         [(node :w:name {:w:val "No List"})
+          (node :w:uiPriority {:w:val "99"})
+          (node :w:semiHidden)
+          (node :w:unhideWhenUsed)])])
+
+(defn when-v-node
   "Returns an XML tag node when the value exists."
   [v tag]
   (cond (nil? v) nil
-        (= "" v) {:tag tag}
-        true {:tag tag
-              :content [(str v)]}))
+        (= "" v) (node tag)
+        true (node tag {} (str v))))

 (defn when-v-kv
   "Returns a vector of [k v] when the value exists."
@@ -195,7 +200,7 @@
   {:tag :properties:Properties
    :attrs doc-props-app-xml-xmlns
    :content (remove-nil
-              (map when-v-tag
+              (map when-v-node
                    [application app-version]
                    [:properties:Application :properties:AppVersion]))})

@@ -234,7 +239,7 @@
   {:tag :cp:coreProperties
    :attrs doc-props-core-xml-xmlns
    :content (remove-nil
-              (map when-v-tag
+              (map when-v-node
                    [title subject creator keywords description last-modified-by]
                    [:dc:title :dc:subject :dc:creator :cp:keywords
                     :dc:description :cp:lastModifiedBy]))})


Make adding a new tag use broadcast-group-change helper

Need to ensure read-only clients (and bots, I guess) see new tags too

--- src/braid/core/server/sync.clj
+++ src/braid/core/server/sync.clj
@@ -202,9 +202,7 @@
           (mapcat
             (fn [u] (tag/user-subscribe-to-tag-txn (u :id) (new-tag :id)))
             users))
-        (doseq [u users]
-          (when (connected? (u :id))
-            (chsk-send! (u :id) [:braid.client/create-tag new-tag])))
+        (broadcast-group-change (:group-id new-tag) [:braid.client/create-tag new-tag])
         (when ?reply-fn
           (?reply-fn {:ok true})))
       (do (timbre/warnf "User %s attempted to create a tag %s with an invalid name"


Refer fewer functions from helper ns

Refer fewer functions from helper ns

--- src/braid/core/server/sync.clj
+++ src/braid/core/server/sync.clj
@@ -28,7 +28,7 @@
     [braid.core.server.search :as search]
     [braid.core.server.socket :refer [chsk-send! connected-uids]]
     [braid.core.server.sync-handler :refer [event-msg-handler]]
-    [braid.core.server.sync-helpers :refer [broadcast-thread broadcast-user-change user-can-message? notify-users]]
+    [braid.core.server.sync-helpers :as helpers :refer [broadcast-group-change]]
     [braid.core.server.uploads :as s3-uploads]
     [braid.core.server.util :refer [valid-url?]]))

@@ -86,12 +86,12 @@
                         (update-in [:content] #(apply str (take 5000 %)))
                         (assoc :created-at (java.util.Date.)))]
     (if (new-message-valid? new-message)
-      (when (user-can-message? user-id new-message)
+      (when (helpers/user-can-message? user-id new-message)
         (db/run-txns! (message/create-message-txn new-message))
         (when-let [cb ?reply-fn]
           (cb :braid/ok))
-        (broadcast-thread (new-message :thread-id) [])
-        (notify-users new-message)
+        (helpers/broadcast-thread (new-message :thread-id) [])
+        (helpers/notify-users new-message)
         (notify-bots new-message))
       (do
         (timbre/warnf "Malformed new message: %s" (pr-str new-message))
@@ -115,7 +115,7 @@
   (let [{:keys [thread-id tag-id]} ?data]
     (let [group-id (tag/tag-group-id tag-id)]
       (db/run-txns! (thread/tag-thread-txn group-id thread-id tag-id))
-      (broadcast-thread thread-id []))))
+      (helpers/broadcast-thread thread-id []))))
     ; TODO do we need to notify-users and notify-bots

@@ -132,7 +132,7 @@
   (if (valid-nickname? (?data :nickname))
     (try
       (do (db/run-txns! (user/set-nickname-txn user-id (?data :nickname)))
-          (broadcast-user-change user-id [:braid.client/name-change
+          (helpers/broadcast-user-change user-id [:braid.client/name-change
                                           {:user-id user-id
                                            :group-ids (map :id (group/user-groups user-id))
                                            :nickname (?data :nickname)}])


Correctly spell "Northern Tier"

Correctly spell "Northern Tier"

--- src/clj/hbg_crime/geometry.clj
+++ src/clj/hbg_crime/geometry.clj
@@ -174,5 +174,5 @@
   {:downtown downtown
    :midtown midtown
    :uptown uptown
-   :norther-tier northern-tier
+   :northern-tier northern-tier
    :allison-hill allison-hill})


initialize cljs.analyzer/*passes* default

--- src/clj/shadow/cljs/passes.clj
+++ src/clj/shadow/cljs/passes.clj
@@ -1,9 +1,5 @@
 (ns shadow.cljs.passes
-  (:require [clojure.pprint :refer (pprint)]
-            [cljs.util :as util]
-            [clojure.string :as str]
-            [clojure.java.io :as io]
-            [cljs.analyzer :as ana]
+  (:require [cljs.analyzer :as ana]
             [cljs.env :as env]))

 (defn load-macros
@@ -11,8 +7,9 @@
    if a macro namespace of the same name as the current ns is found,
    we extract all the names of all macros so later passes have an easier time
    figuring out what might be a macro"
-  [_ {:keys [op name require-macros] :as ast}]
-  (let [require-macros (vals require-macros)]
+  [_ {:keys [op name require-macros use-macros] :as ast}]
+  (let [require-macros (concat (vals require-macros)
+                               (vals use-macros))]
     (doseq [macro-ns require-macros]
       (require macro-ns))

@@ -88,3 +85,11 @@
               (ana/error env (ana/error-message :undeclared-ns-form {:type "var" :lib lib :sym sym})))))
         ast)))

+;; set default passes here since cljs.analyzer cannot do this (it doesn't know this ns)
+(alter-var-root #'ana/*passes*
+                (fn [_]
+                  [load-macros
+                   infer-macro-require
+                   infer-macro-use
+                   check-uses
+                   ana/infer-tag]))


Stop process before checking version and check plugins too

--- src/clojure/nightcode/lein.clj
+++ src/clojure/nightcode/lein.clj
@@ -287,7 +287,7 @@

 (defn check-versions-in-project-task
   [path project]
-  (leiningen.ancient/ancient project ":no-colors"))
+  (leiningen.ancient/ancient project ":all" ":no-profiles" ":no-colors"))

 (defn hot-swap-project-task
   [path project]
@@ -360,6 +360,7 @@

 (defn check-versions-in-project
   [process in out path]
+  (stop-process process)
   (->> (do (println (utils/get-string :checking_versions))
          (start-process-directly process path check-versions-in-project-task))
        (start-thread in out)))


Always delegate dux sinks to `wrap-sink`.

--- src/clojure/parkour/io/dux.clj
+++ src/clojure/parkour/io/dux.clj
@@ -82,27 +82,17 @@
         tac (mr/tac conf context), c (get-counter context oname)
         ckey (.getOutputKeyClass job), cval (.getOutputValueClass job)
         rw (.getRecordWriter ^OutputFormat of tac)]
-    (reify
-      Configurable
-      (getConf [_] conf)
-
-      w/Wrapper
-      (unwrap [_] rw)
-
-      snk/TupleSink
-      (-key-class [_] ckey)
-      (-val-class [_] cval)
-      (-emit-keyval [_ key val]
-        (.write rw key val)
-        (.increment c 1))
-
-      IFn
-      (invoke [sink keyval] (snk/emit-keyval sink keyval))
-      (invoke [sink key val] (snk/emit-keyval sink key val))
-      (applyTo [sink args]
-        (case (count args)
-          1 (let [[keyval] args] (snk/emit-keyval sink keyval))
-          2 (let [[key val] args] (snk/emit-keyval sink key val)))))))
+    (snk/wrap-sink
+     (reify
+       Configurable (getConf [_] conf)
+       w/Wrapper (unwrap [_] rw)
+       snk/TupleSink
+       (-key-class [_] ckey)
+       (-val-class [_] cval)
+       (-close [_] (.close rw context))
+       (-emit-keyval [_ key val]
+         (.write rw key val)
+         (.increment c 1))))))

 (defn get-sink
   "Get sink for named output `oname` and optional (file output format only) file
@@ -112,7 +102,6 @@
      (let [[jobs ofs rws] (dux-state context), rwkey [oname base]]
        @(or (get-in @rws rwkey)
             (let [new-rw (partial new-rw context oname base)
-                  new-rw (comp snk/wrap-sink new-rw)
                   add-rw (fn [rws]
                            (if rws
                              (if-let [rw (get-in rws rwkey)]


Fixed bug with posting expenses

Fixed bug with posting expenses

--- src/clojure/skuld/rest_api.clj
+++ src/clojure/skuld/rest_api.clj
@@ -12,9 +12,11 @@
 (defresource groups-resource [db id]
   :available-media-types ["application/json"]
   :handle-ok (fn [ctx]
-               (json/write-str (groups/get-group db id))))
+               (if (nil? id)
+                 (json/write-str (groups/list-groups db))
+                 (json/write-str (groups/get-group db id)))))

-(defresource expenses-resourse [group-id]
+(defresource expenses-resourse [db group-id]
   :allowed-methods [:post :get]
   :available-media-types ["application/json" "application/clojure"]
   :handle-ok (fn [ctx]
@@ -24,18 +26,18 @@
            (let [body (slurp (-> ctx :request :body))
                  data (json/read-str body :key-fn keyword)
                  group-id (get-in ctx [:request :params :id])
-                 id (groups/add-shared-expense group-id (:payed-by data)
+                 id (groups/add-shared-expense db group-id (:payed-by data)
                                                (:amount data) (:shared-with data))]
              {::expense-id id})))

-(defresource dept-resource [group-id]
+(defresource dept-resource [db group-id]
   :allowed-methods [:get]
   :available-media-types ["application/json" "application/clojure"]
   :handle-ok (fn [ctx]
                (let [media-type (get-in ctx [:representation :media-type])]
                  (condp = media-type
-                   "application/json" (into [] (groups/get-group-dept database group-id))
-                   (into [] (groups/get-group-dept database group-id))))))
+                   "application/json" (into [] (groups/get-group-dept db group-id))
+                   (into [] (groups/get-group-dept db group-id))))))

 (defn- hello-world [request]
   (-> (res/response "Hello, api!")
@@ -44,9 +46,10 @@
 (defn api-routes
   [db]
   (routes
+    (ANY "/groups" [] (groups-resource db nil))
     (ANY "/groups/:id" [id] (groups-resource db id))
-    (ANY "/groups/:id/expenses" [id] (expenses-resourse id))
-    (ANY "/groups/:id/dept" [id] (dept-resource id))))
+    (ANY "/groups/:id/expenses" [id] (expenses-resourse db id))
+    (ANY "/groups/:id/dept" [id] (dept-resource db id))))

 (def api-handler
   (-> (api-routes database)


Added whole-text-files to load files as a single record with a key, also supports the min-partitions option as well.

--- src/clojure/sparkling/core.clj
+++ src/clojure/sparkling/core.clj
@@ -114,6 +114,15 @@
   [spark-context filename]
   (.textFile spark-context filename))

+(defn whole-text-files
+  "Read a directory of text files from HDFS, a local file system (available on all nodes),
+  or any Hadoop-supported file system URI. Each file is read as a single record and returned
+  in a key-value pair, where the key is the path of each file, the value is the content of each file."
+  ([spark-context filename min-partitions]
+   (.wholeTextFiles spark-context filename min-partitions))
+  ([spark-context filename]
+   (whole-text-files spark-context filename 8)))
+
 (defn into-rdd
   "Distributes a local collection to form/return an RDD"
   ([spark-context lst] (.parallelize spark-context lst))


Changing reset status code.

Changing reset status code.

--- src/cmr/indexer/api/routes.clj
+++ src/cmr/indexer/api/routes.clj
@@ -43,7 +43,7 @@
       (let [context (acl/add-authentication-to-context request-context params headers)]
         (acl/verify-ingest-management-permission context :update)
         (index-svc/reset context))
-      {:status 200})
+      {:status 204})

     ;; Sends an update to the index set to update mappings and index settings.
     (POST "/update-indexes" {:keys [request-context params headers]}
@@ -73,7 +73,7 @@
         (let [ignore-conflict (ignore-conflict? params)
               context (acl/add-authentication-to-context context params headers)]
           (index-svc/delete-concept context concept-id revision-id ignore-conflict)
-          (r/response nil))))
+          {:status 204})))

     (route/not-found "Not Found")))


Better doc for "are" macro

Better doc for "are" macro

--- src/main/clojure/com/stuartsierra/lazytest.clj
+++ src/main/clojure/com/stuartsierra/lazytest.clj
@@ -174,7 +174,7 @@
           3 2  5
           8 -1 7)

-      ;; Compiles to:
+      ;; Equivalent to:

       (is (= (+ 2 2) 4)
           (= (+ 3 2) 5)


Make :pedantic work ok

Make :pedantic work ok

--- link/.lein/profiles.clj
+++ link/.lein/profiles.clj
@@ -2,7 +2,8 @@
         :plugins [[cider/cider-nrepl "0.9.1"]
                   [refactor-nrepl "1.1.0"]
                   [lein-pprint "1.1.1"]
-                  [lein-bikeshed "0.2.0"]]
+                  [lein-bikeshed "0.2.0"]
+                  [lein-ancient "0.6.7" :exclusions [org.clojure/clojure]]]
         :source-paths ["/home/vagrant/.lein/src"]
         :injections [;; humane-test-output makes clojure.test output more better
                      (require 'pjstadig.humane-test-output)
@@ -11,7 +12,7 @@
                      ;; Spyscope is for tracing.
                      (require 'spyscope.core)]
         :dependencies [[org.clojure/tools.namespace "0.2.11"]
-                       [org.clojure/tools.nrepl "0.2.7"]
+                       [org.clojure/tools.nrepl "0.2.10"]
                        [org.clojure/tools.trace "0.7.8"]
                        [pjstadig/humane-test-output "0.7.0"]
                        [slamhound "1.5.5"]


Got tests passing by adding login to ES store defrecords, but still not using the login values

--- src/cia/stores/es/store.clj
+++ src/cia/stores/es/store.clj
@@ -23,7 +23,7 @@

 (defrecord JudgementStore [state]
   IJudgementStore
-  (create-judgement [_ new-judgement]
+  (create-judgement [_ login new-judgement]
     (ju/handle-create-judgement state new-judgement))
   (read-judgement [_ id]
     (ju/handle-read-judgement state id))
@@ -36,16 +36,16 @@

 (defrecord FeedbackStore [state]
   IFeedbackStore
-  (create-feedback [_ new-feedback judgement-id]
+  (create-feedback [_ new-feedback login judgement-id]
     (fe/handle-create-feedback state new-feedback judgement-id))
   (list-feedback [_ filter-map]
     (fe/handle-list-feedback state filter-map)))

 (defrecord IndicatorStore [state]
   IIndicatorStore
-  (create-indicator [_ new-indicator]
+  (create-indicator [_ login new-indicator]
     (in/handle-create-indicator state new-indicator))
-  (update-indicator [_ id new-indicator]
+  (update-indicator [_ id login new-indicator]
     (in/handle-update-indicator state id new-indicator))
   (read-indicator [_ id]
     (in/handle-read-indicator state id))
@@ -67,9 +67,9 @@
   ITTPStore
   (read-ttp [_ id]
     (ttp/handle-read-ttp state id))
-  (create-ttp [_ new-ttp]
+  (create-ttp [_ login new-ttp]
     (ttp/handle-create-ttp state new-ttp))
-  (update-ttp [_ id new-ttp]
+  (update-ttp [_ id login new-ttp]
     (ttp/handle-update-ttp state id new-ttp))
   (delete-ttp [_ id]
     (ttp/handle-delete-ttp state id))
@@ -80,9 +80,9 @@
   IActorStore
   (read-actor [_ id]
     (ac/handle-read-actor state id))
-  (create-actor [_ new-actor]
+  (create-actor [_ login new-actor]
     (ac/handle-create-actor state new-actor))
-  (update-actor [_ id actor]
+  (update-actor [_ id login actor]
     (ac/handle-update-actor state id actor))
   (delete-actor [_ id]
     (ac/handle-delete-actor state id))
@@ -93,9 +93,9 @@
   ICampaignStore
   (read-campaign [_ id]
     (ca/handle-read-campaign state id))
-  (create-campaign [_ new-campaign]
+  (create-campaign [_ login new-campaign]
     (ca/handle-create-campaign state new-campaign))
-  (update-campaign [_ id new-campaign]
+  (update-campaign [_ id login new-campaign]
     (ca/handle-update-campaign state id new-campaign))
   (delete-campaign [_ id]
     (ca/handle-delete-campaign state id))
@@ -106,9 +106,9 @@
   ICOAStore
   (read-coa [_ id]
     (coa/handle-read-coa state id))
-  (create-coa [_ new-coa]
+  (create-coa [_ login new-coa]
     (coa/handle-create-coa state new-coa))
-  (update-coa [_ id new-coa]
+  (update-coa [_ id login new-coa]
     (coa/handle-update-coa state id new-coa))
   (delete-coa [_ id]
     (coa/handle-delete-coa state id))
@@ -119,9 +119,9 @@
   IIncidentStore
   (read-incident [_ id]
     (inc/handle-read-incident state id))
-  (create-incident [_ new-incident]
+  (create-incident [_ login new-incident]
     (inc/handle-create-incident state new-incident))
-  (update-incident [_ id new-incident]
+  (update-incident [_ id login new-incident]
     (inc/handle-update-incident state id new-incident))
   (delete-incident [_ id]
     (inc/handle-delete-incident state id))
@@ -132,9 +132,9 @@
   IExploitTargetStore
   (read-exploit-target [_ id]
     (et/handle-read-exploit-target state id))
-  (create-exploit-target [_ new-exploit-target]
+  (create-exploit-target [_ login new-exploit-target]
     (et/handle-create-exploit-target state new-exploit-target))
-  (update-exploit-target [_ id new-exploit-target]
+  (update-exploit-target [_ id login new-exploit-target]
     (et/handle-update-exploit-target state id new-exploit-target))
   (delete-exploit-target [_ id]
     (et/handle-delete-exploit-target state id))


fix page export

fix page export

--- src/clj/salava/page/pdf.clj
+++ src/clj/salava/page/pdf.clj
@@ -11,6 +11,7 @@
             [clojure.zip :as zip]
             [salava.user.db :as ud]
             [net.cgrand.enlive-html :as enlive]
+            [salava.badge.pdf :refer [process-markdown replace-nils]]
             ))

 (defqueries "sql/page/main.sql")
@@ -68,15 +69,14 @@
        ;;     (apply str)
        ))

-
 (defn generate-pdf [ctx page-id user-id header?]
   (let [ badge-info-fn (first (plugin-fun (get-plugins ctx) "pdf" "pdf-generator-helper"))
-         page (conj () (page-with-blocks ctx page-id))
+         page (replace-nils (conj () (page-with-blocks ctx page-id)))
          data-dir (get-in ctx [:config :core :data-dir])
          site-url (get-in ctx [:config :core :site-url])
          plugins (get-plugins ctx)
          user-data (ud/user-information ctx user-id)
-         ul (:language user-data)
+         ul (if (blank? (:language user-data)) "en" (:language user-data))
          font-path  (first (mapcat #(get-in ctx [:config % :font] []) plugins))
          header-path (first (mapcat #(get-in ctx [:config % :logo] []) plugins))
          font  {:ttf-name (str site-url font-path)}
@@ -99,18 +99,20 @@
                                                                (case (:size %)
                                                                  "h1" [:paragraph.generic {:align :left }
                                                                        [:spacer 0]
-                                                                       [:heading (or (:content %) "")]
+                                                                       [:heading  (:content %)]
                                                                        [:line {:dotted true}]
                                                                        ]
                                                                  "h2" [:paragraph.generic {:align :left}
                                                                        [:spacer 1]
-                                                                       [:heading {:style {:size 12 :align :center}}  (or (:content %) "")]
+                                                                       [:heading {:style {:size 12 :align :center}}  (:content %)]
                                                                        [:line {:dotted true}]] ))

                                                              (when (= "badge" (:type %))
                                                                [:table {:widths [1 3] :border false :keep-together? false }
+
                                                                 [[:cell {:align :center}
-                                                                  (when-not (blank? (:image_file %))
+                                                                  (if (= "-" (:image_file %))
+                                                                    ""
                                                                     [:paragraph {:align :center :keep-together true}
                                                                      (if (ends-with? (:image_file %) "png")
                                                                        [:paragraph [:chunk [:image {:align :center :width 80 :height 80} (str data-dir "/" (:image_file %))]] [:spacer 1]"\n" ]
@@ -119,33 +121,30 @@
                                                                    (let [badge (badge-info-fn ctx user-id (conj () (:badge_id %)))]
                                                                     (when-not (empty? badge)
                                                                       [:phrase {:align :center}
-                                                                       [:chunk [:image {:align :center :width 75 :height 75  :base64 true} (or (:qr_code (first badge)) "")]]"\n"
+                                                                       [:chunk [:image {:align :center :width 75 :height 75  :base64 true} (:qr_code (first badge))]]"\n"
 ;;                                                                        [:spacer 0]
-                                                                       [:chunk.generic (t :badge/Scantobadge ul)]]))]
-                                                                  ]
+                                                                       [:chunk.generic (t :badge/Scantobadge ul)]]))]]

                                                                  [:cell
-                                                                  [:heading.generic (or (:name %) "-")]
+                                                                  [:heading.generic  (:name %)]
                                                                   [:spacer 0]
                                                                   [:paragraph.generic
-                                                                   [:chunk.bold (str (t :badge/Issuedby ul) ": ")] [:chunk (or (:issuer_content_name %) "-")] "\n"
-                                                                   [:chunk.bold (str (t :badge/Issuedon ul)": ")] [:chunk (date-from-unix-time (long (* 1000 (:issued_on %))) "date")]
+                                                                   [:chunk.bold (str (t :badge/Issuedby ul) ": ")] [:chunk (:issuer_content_name %)] "\n"
+                                                                   [:chunk.bold (str (t :badge/Issuedon ul)": ")] [:chunk (if (number? (:issued_on %)) (date-from-unix-time (long (* 1000 (:issued_on %))) "date") (:issued_on %))]
                                                                    [:spacer 0]
-                                                                   (or (:description %) "-") "\n"
+                                                                   (:description %)"\n"
                                                                    [:spacer 0]
                                                                    [:paragraph {:keep-together true}
                                                                     [:phrase.bold (str (t :badge/Criteria ul)": ")] [:spacer 0]
-                                                                    [:anchor {:target (:criteria_url %) :style{:family :times-roman :color [66 100 162]}} (or (:criteria_url %) "-")]
+                                                                    [:anchor {:target (:criteria_url %) :style{:family :times-roman :color [66 100 162]}} (:criteria_url %)]
                                                                     [:spacer 0]
-                                                                    (let [badge (badge-info-fn ctx user-id (conj () (:badge_id %)))
-                                                                          content (first (:content (first badge)))]
-                                                                      (when-not (blank? (:criteria_content content))
-                                                                        (markdown->clj-pdf {:paragraph {:keep-together? false} :spacer {:allow-extra-line-breaks? false} :wrap {:global-wrapper :paragraph}} (:criteria_content content))
-                                                                        ))]]]]
+                                                                    (let [content (some->> (badge-info-fn ctx user-id (conj () (:badge_id %))) first :content first) ]
+                                                                      (when-not (empty? content)
+                                                                        (process-markdown "Criteria" (:criteria_content content))))]]]]

                                                                 [[:cell {:colspan 2} [:line {:dotted true}]]]])

-                                                             (when (and (= "html" (:type %)) (not (blank? (:content %))))
+                                                             (when (and (= "html" (:type %)) (not (= "-" (:content %))))
                                                                [:paragraph.generic {:keep-together false :align :left}
                                                                 [:chunk.bold {:size 11 :style :bold} (str (t :page/html ul) ": ")]"\n"
                                                                 [:spacer 1]
@@ -171,23 +170,21 @@
                                                                  (into [:table {:border false :widths [1 3] :keep-together? false}]
                                                                        (conj  (into [[[:cell {:colspan 2}]]] (for [badge (:badges %)
                                                                                :let [b (badge-info-fn ctx user-id (conj () (:id badge)))
-                                                                                content (first (:content (first b)))]]
+                                                                                content (-> b first :content first)]]
                                                                                (when-not (empty? b)
-                                                                                (conj [[:cell {:align :left} (if (ends-with? (:image_file badge) "png") [:image {:align :center :width 75 :height 75}(str data-dir "/" (:image_file badge))] [:image {:align :center :width 75 :height 75 :base64 true} (:qr_code (first b))])]]
+                                                                                (conj [[:cell {:align :left} (if (and (not (= "-" (:image_file badge))) (ends-with? (:image_file badge) "png")) [:image {:align :center :width 75 :height 75}(str data-dir "/" (:image_file badge))] [:image {:align :center :width 75 :height 75 :base64 true} (:qr_code (first b))])]]
                                                                                [:cell
                                                                                [:paragraph.generic
                                                                                [:heading (:name badge)]
-                                                                                [:chunk.bold (str (t :badge/Issuedby ul) ": ")] [:chunk (or (:issuer_content_name content) "-")] "\n"
-                                                                                (when-not (blank? (str (:issued_on badge)))
+                                                                                [:chunk.bold (str (t :badge/Issuedby ul) ": ")] [:chunk (:issuer_content_name content)] "\n"
                                                                                [:phrase
-                                                                                [:chunk.bold (str (t :badge/Issuedon ul)": ")] [:chunk (date-from-unix-time (long (* 1000 (:issued_on badge))) "date")] "\n"])
-                                                                                (or (:description badge) "-") "\n"
+                                                                                [:chunk.bold (str (t :badge/Issuedon ul)": ")] [:chunk (if (number? (:issued_on badge)) (date-from-unix-time (long (* 1000 (:issued_on badge))) "date") (:issued_on badge))] "\n"]
+                                                                                (:description badge)"\n"
                                                                                [:spacer 0]
                                                                                [:paragraph {:keep-together true}
                                                                                [:phrase.bold (str (t :badge/Criteria ul)": ")] "\n"
-                                                                                [:anchor {:target (:criteria_url badge) :style{:family :times-roman :color [66 100 162]}} (or (:criteria_url badge) "-")] "\n"
-                                                                                (when-not (blank? (:criteria_content content))
-                                                                                (markdown->clj-pdf {:spacer {:extra-starting-value 0 :allow-extra-line-breaks? false} :paragraph {:keep-together? true} :wrap {:global-wrapper :paragraph}} (or (:criteria_content content) "-")))
+                                                                                [:anchor {:target (:criteria_url badge) :style{:family :times-roman :color [66 100 162]}} (:criteria_url badge)] "\n"
+                                                                                (process-markdown "Criteria" (:criteria_content content))
                                                                                ]
                                                                                ][:spacer 0]]))))
                                                                               [[:cell {:colspan 2}[:line {:dotted true}]]]))
@@ -197,14 +194,13 @@
                                                                     [:table {:align :center :width-percent 100 :border false :num-cols 5}
                                                                      (into [[:cell {:colspan 5}]] (for [badge (:badges %)
                                                                                :let [b (badge-info-fn ctx user-id (conj () (:id badge)))
-                                                                                content (first (:content (first b)))]]
+                                                                                content (-> b first :content first)]]
                                                                                (when-not (empty? b)
                                                                                [:cell {:padding-right 10}[:anchor {:target (str site-url "/badge/info/" (:id badge))}
                                                                                (if (ends-with? (:image_file badge) "png") [:chunk [:image {:align :center :width 60 :height 60}(str data-dir "/" (:image_file badge))]] [:chunk [:image {:align :center :width 60 :height 60 :base64 true} (:qr_code (first b))]])
                                                                                ]])))]]]
                                                                   [[:cell [:line {:dotted true}]]]]))
-                                                             [:spacer 3]
-                                                             ])
+                                                             [:spacer 3]])

                                content (-> (mapv template $blocks)
                                            (conj   [[:pdf-table {:align :right :width-percent 100 :cell-border false}
@@ -224,7 +220,6 @@
                            ))
          ]
     ;; TODO FIX HTML
-
     (fn [out]
       (pdf/pdf (into [pdf-settings [:spacer 4]]
                      (page-template page)) out))))


add html field to markup description

add html field to markup description

not currently used and might not be necessary/practical/possible to
use. How can we build om/dom elements from this data?

--- src/cljs/greffe/markup.cljs
+++ src/cljs/greffe/markup.cljs
@@ -3,23 +3,30 @@

 (def markup
   {:body {:type :container
-          :contains [:div]}
+          :contains [:div]
+          :html "div"}
    :text {:type :container
-          :contains [:div]}
+          :contains [:div]
+          :html "div"}
    :div {:type :container
-         :contains [:div :head :lg]}
+         :contains [:div :head :lg]
+         :html "div"}
    :head {:type :chunk
           :contains [:note]
-          :line-token "*"}
+          :line-token "*"
+          :html "h2"}
    :lg {:type :multi-chunk
-        :contains [:l]}
+        :contains [:l]
+        :html "ul"}
    :l {:type :chunk
        :contains [:rhyme :caesura :note]
-       :line-token "-"}
+       :line-token "-"
+       :html "li"}
    :note {:type :inner
           :begin-token "{{"
           :end-token "}}"
-          :contains []}
+          :contains []
+          :html "strong"}
    :caesura {:type :empty
              :begin-token "|"
              :end-token "|"