Added simulation code to assist in running hebbian simulations efficiently.

--- src/main/clojure/brainardlab/nben/retina/simulation.clj
+++ src/main/clojure/brainardlab/nben/retina/simulation.clj
@@ -212,6 +212,32 @@
                                                                        0 R a save-every)
                                           :full-reduce false)))

+(defn write-hebbian-simulation
+  "(write-hebbian-simulation output-filename retina hebbian-sim-result) writes information about
+   the given retina and result data produced by the Hebbian simulation to the file named by
+   output-filename."
+  [output-filename R H]
+  (let [[X V M] H
+        nmem (count (first M))
+        labels (get R :labels)
+        n (count labels)
+        coords (get (get R :mosaic) :coordinates)
+        rmap (loop [r {}, s (seq (get R :cones)), k 0]
+               (if s
+                 (recur (assoc r (first s) k) (next s) (inc k))
+                 r))]
+    (with-open [o (java.io.DataOutputStream. (java.io.FileOutputStream. output-filename))]
+      (.writeInt o n)
+      (.writeInt o nmem)
+      ;; cone types
+      (doseq [l labels] (.writeInt o (get rmap l)))
+      ;; coordinates
+      (doseq [c coords, x c] (.writeFloat o (float x)))
+      ;; hebbian sim stuff
+      (doseq [x X] (.writeFloat o (float (nth x 2))))
+      (doseq [v V] (.writeFloat o (float v)))
+      (doseq [m M, v m] (.writeFloat o (float v))))))
+
 (defn- hebbian-analysis [signals cones opts]
   (cond
    ;; end of analysis...
@@ -232,65 +258,70 @@
          s (map #(/ % max-s) pre-s)
          steps (get opts :steps-per-signal) ;; how many steps we run each signal
          nmem (get opts :memory) ;; the amount of memory for each cone
-         k0 (get opts :k)
-         k (mod k0 nmem) ;; the place we're putting this signal in the memory
+         k (mod (get opts :k) nmem) ;; the place we're putting this signal in the memory
          dfn (get opts :distance-fn) ;; ideal distance between cones by response angle cos
          strength (get opts :spring-strength) ;; strength of a spring
          dampen (- 1.0 (get opts :dampen))
          dt (get opts :timestep)
          dt2 (* dt dt)
          [startX startV M] cones
-         step-mon (get opts :step-monitor)]
+         step (get opts :step)]
+     (when (and (= 0 (mod step 10)) (:verbose opts))
+       (println "Step: " step)
+       (when (= 0 (mod step 10))
+         (when-let [save-dir (get opts :save-dir)]
+           (write-hebbian-simulation (format "%s/hebbian%07d.bin" save-dir step)
+                                     (:retina opts)
+                                     [startX startV M]))))
      ;; update memory...
-     (doall (map #(aset %1 k (float %2)) M s))
+     (doall (pmap #(aset %1 k (float %2)) M s))
      ;; make a similarity matrix (ideal distances)
      (let [S (let [norms (map (fn [m] (sqrt (reduce #(+ %1 (* %2 %2)) 0 m))) M)]
-               (vec (map (fn [m1 n1]
-                           (vec (map (fn [m2 n2]
-                                       (dfn (/ (reduce + (map * m1 m2)) (* n1 n2))))
-                                     M norms)))
-                         M norms)))]
-       (if step-mon (step-mon k0 startX startV))
+               (vec (pmap (fn [m1 n1]
+                            (vec (map (fn [m2 n2]
+                                        (dfn (/ (reduce + (map * m1 m2)) (* n1 n2))))
+                                      M norms)))
+                          M norms)))]
        ;; recur (after running the simulation loops)
        (recur
         (next signals)
         (loop [X0 startX, V0 startV, iter 0]
           (let [;; directions and distances between cones
-                D (vec (map (fn [x1]
-                              (vec (map (fn [x2]
-                                          (let [v (map - x2 x1)
-                                                d (sqrt (reduce #(+ %1 (* %2 %2)) 0 v))]
-                                            [d (if (or (= x1 x2) (= 0 d))
-                                                 0
-                                                 (/ (nth v 2) d))]))
-                                        X0)))
-                            X0))
+                D (vec (pmap (fn [x1]
+                               (vec (map (fn [x2]
+                                           (let [v (map - x2 x1)
+                                                 d (sqrt (reduce #(+ %1 (* %2 %2)) 0 v))]
+                                             [d (if (or (= x1 x2) (= 0 d))
+                                                  0
+                                                  (/ (nth v 2) d))]))
+                                         X0)))
+                             X0))
                 ;; get the accelerations
-                A (doall (map (fn [drow srow]
-                                (reduce + (map (fn [[d u] s] (* strength (- d s) u))
-                                               drow srow)))
-                              D S))
+                A (doall (pmap (fn [drow srow]
+                                 (reduce + (map (fn [[d u] s] (* strength (- d s) u))
+                                                drow srow)))
+                               D S))
                 ;; now, we update positions...
-                X (doall (map (fn [x0 v a]
-                                (assoc x0 2 (+ (nth x0 2) (* v dt) (* 0.5 dt2 a))))
+                X (doall (pmap (fn [x0 v a]
+                                 (assoc x0 2 (+ (nth x0 2) (* v dt) (* 0.5 dt2 a))))
                                         ;(vec (map #(+ %1 (* dt %2) (* dt2 0.5 %3))
                                         ;          x0 v a)))
-                              X0 V0 A))
+                               X0 V0 A))
                 ;; and the velocities
-                V (doall (map (fn [v0 a] (+ (* dampen v0) (* dt a)))
-                              V0 A))]
+                V (doall (pmap (fn [v0 a] (+ (* dampen v0) (* dt a)))
+                               V0 A))]
             ;; if we've done enough iterations, yield the result, otherwise, continue
             (if (< iter steps)
               (recur X V (inc iter))
               [X V M])))
-        (assoc opts :k (inc k)))))))
+        (assoc opts :k (inc k) :step (inc step)))))))

 (defn simulate-retinas-hebbian
   "Yields a simulation of the retinas (using simulate-retinas), but wraps the call such that all
    analysis is done using Hebbian learning implemeted by spring simulation."
   [retinas hs-database & {:keys [memory image-count dampen timestep steps-per-signal
-                                 spring-strength distance-fn step-monitor]
-                          :or {memory 20 image-count 1000 dampen 0.05 timestep 0.01
+                                 spring-strength distance-fn verbose save-dir]
+                          :or {memory 20 image-count 1000 dampen 0.05 timestep 0.001
                                steps-per-signal 10 spring-strength 1
                                distance-fn (let [c (Math/exp -2.0)]
                                              #(+ 2.0 (* 20.0 (- (Math/exp (* -2.0 %)) c))))}}]
@@ -310,7 +341,10 @@
                                        :steps-per-signal steps-per-signal
                                        :spring-strength spring-strength
                                        :distance-fn distance-fn
-                                       :step-monitor step-monitor
+                                       :verbose verbose
+                                       :save-dir save-dir
+                                       :retina ret1
+                                       :step 0
                                        :k 0})
      :full-reduce false)))

@@ -380,29 +414,3 @@
           (doseq [coord bmuCoords, x coord] (.writeFloat o (float x)))
           (doseq [nrn weights, w nrn] (.writeFloat o (float w))))))))

-(defn write-hebbian-simulation
-  "(write-hebbian-simulation output-filename retina hebbian-sim-result) writes information about
-   the given retina and result data produced by the Hebbian simulation to the file named by
-   output-filename."
-  [output-filename R H]
-  (let [[X V M] H
-        nmem (count (first M))
-        labels (get R :labels)
-        n (count labels)
-        coords (get (get R :mosaic) :coordinates)
-        rmap (loop [r {}, s (seq (get R :cones)), k 0]
-               (if s
-                 (recur (assoc r (first s) k) (next s) (inc k))
-                 r))]
-    (with-open [o (java.io.DataOutputStream. (java.io.FileOutputStream. output-filename))]
-      (.writeInt o n)
-      (.writeInt o nmem)
-      ;; cone types
-      (doseq [l labels] (.writeInt o (get rmap l)))
-      ;; coordinates
-      (doseq [c coords, x c] (.writeFloat o (float x)))
-      ;; hebbian sim stuff
-      (doseq [x X] (.writeFloat o (float (nth x 2))))
-      (doseq [v V] (.writeFloat o (float v)))
-      (doseq [m M, v m] (.writeFloat o (float v))))))
-


Riemann only accepts strings as tags

Riemann only accepts strings as tags

--- src/onyx/lifecycle/metrics/metrics.clj
+++ src/onyx/lifecycle/metrics/metrics.clj
@@ -34,6 +34,7 @@
         name (str (:metrics/workflow-name lifecycle))
         task-name (str (:onyx.core/task event))
         peer-id (:onyx.core/id event)
+        peer-id-str (str peer-id)
         core {:job-id (:onyx.core/job-id event)
               :task-id (:onyx.core/task-id event)
               :task-name (:onyx.core/task event)
@@ -63,32 +64,32 @@
                                     :window "1s"
                                     :metric :throughput
                                     :value (apply + (remove nil? (take 1 throughputs-val)))
-                                    :tags ["throughput_1s" "onyx" task-name name peer-id]}))
+                                    :tags ["throughput_1s" "onyx" task-name name peer-id-str]}))

                (>!! ch (merge core {:service (format "[%s] 10s_throughput" task-name)
                                     :window "10s"
                                     :metric :throughput
                                     :value (apply + (remove nil? (take 10 throughputs-val)))
-                                    :tags ["throughput_10s" "onyx" task-name name peer-id]}))
+                                    :tags ["throughput_10s" "onyx" task-name name peer-id-str]}))

                (>!! ch (merge core {:service (format "[%s] 60s_throughput" task-name)
                                     :window "60s"
                                     :metric :throughput
                                     :value (apply + (remove nil? (take 60 throughputs-val)))
-                                    :tags ["throughput_60s" "onyx" task-name name peer-id]})))
+                                    :tags ["throughput_60s" "onyx" task-name name peer-id-str]})))

                (let [retry-rate-val (im/snapshot! (:retry-rate metrics))]
                  (>!! ch (merge core {:service (format "[%s] 1s_retry-segment-rate" task-name)
                                       :window "1s"
                                       :metric :retry-rate
                                       :value retry-rate-val
-                                      :tags ["retry_segment_rate_1s" "onyx" task-name name peer-id]})))
+                                      :tags ["retry_segment_rate_1s" "onyx" task-name name peer-id-str]})))

                  (when (= :input (:onyx/type (:onyx.core/task-map event)))
                    (>!! ch (merge core {:service (format "[%s] pending_messages_count" task-name)
                                         :metric :pending-messages-count
                                         :value @pending-size
-                                        :tags ["pending_messages_count" "onyx" task-name name peer-id]}))
+                                        :tags ["pending_messages_count" "onyx" task-name name peer-id-str]}))

                    (when (= cycle-count 0) ; only snapshot every 10s
                      (let [completion-rate+latency (:completion-rate+latencies-10s metrics)
@@ -105,34 +106,34 @@
                                             :metric :complete-latency
                                             :quantile 0.50
                                             :value (get latencies-vals 0.5)
-                                            :tags ["complete_latency_50th" "onyx" "50_percentile" task-name name peer-id]}))
+                                            :tags ["complete_latency_50th" "onyx" "50_percentile" task-name name peer-id-str]}))

                        (>!! ch (merge core {:service (format "[%s] 90_0th_percentile_complete_latency" task-name)
                                             :window "10s"
                                             :quantile 0.90
                                             :metric :complete-latency
                                             :value (get latencies-vals 0.90)
-                                            :tags ["complete_latency_90th" "onyx" task-name name peer-id]}))
+                                            :tags ["complete_latency_90th" "onyx" task-name name peer-id-str]}))

                        (>!! ch (merge core {:service (format "[%s] 99_0th_percentile_complete_latency" task-name)
                                             :window "10s"
                                             :quantile 0.99
                                             :metric :complete-latency
                                             :value (get latencies-vals 0.99)
-                                            :tags ["complete_latency_99th" "onyx" task-name name peer-id]}))
+                                            :tags ["complete_latency_99th" "onyx" task-name name peer-id-str]}))

                        (>!! ch (merge core {:service (format "[%s] 99_9th_percentile_complete_latency" task-name)
                                             :window "10s"
                                             :quantile 0.999
                                             :metric :complete-latency
                                             :value (get latencies-vals 0.999)
-                                            :tags ["complete_latency_99_9th" "onyx" task-name name peer-id]}))
+                                            :tags ["complete_latency_99_9th" "onyx" task-name name peer-id-str]}))
                        (>!! ch (merge core {:service (format "[%s] max_complete_latency" task-name)
                                             :window "10s"
                                             :quantile 1.0
                                             :metric :complete-latency
                                             :value (get latencies-vals 1.0)
-                                            :tags ["complete_latency_max" "onyx" task-name name peer-id]})))))
+                                            :tags ["complete_latency_max" "onyx" task-name name peer-id-str]})))))

                  (let [rate+latency (:rate+latency-10s metrics)
                        latency-snapshot (im/snapshot! rate+latency)
@@ -147,34 +148,34 @@
                                         :quantile 0.50
                                         :metric :batch-latency
                                         :value (get latencies-vals 0.5)
-                                        :tags ["batch_latency_50th" "onyx" "50_percentile" task-name name peer-id]}))
+                                        :tags ["batch_latency_50th" "onyx" "50_percentile" task-name name peer-id-str]}))

                    (>!! ch (merge core {:service (format "[%s] 90th_percentile_batch_latency" task-name)
                                         :window "10s"
                                         :quantile 0.90
                                         :value (get latencies-vals 0.90)
                                         :metric :batch-latency
-                                        :tags ["batch_latency_90th" "onyx" task-name name peer-id]}))
+                                        :tags ["batch_latency_90th" "onyx" task-name name peer-id-str]}))

                    (>!! ch (merge core {:service (format "[%s] 99th_percentile_batch_latency" task-name)
                                         :window "10s"
                                         :quantile 0.99
                                         :value (get latencies-vals 0.99)
                                         :metric :batch-latency
-                                        :tags ["batch_latency_99th" "onyx" task-name name peer-id]}))
+                                        :tags ["batch_latency_99th" "onyx" task-name name peer-id-str]}))

                    (>!! ch (merge core {:service (format "[%s] 99_9th_percentile_batch_latency" task-name)
                                         :window "10s"
                                         :quantile 0.999
                                         :value (get latencies-vals 0.999)
                                         :metric :batch-latency
-                                        :tags ["batch_latency_99_9th" "onyx" task-name name peer-id]}))
+                                        :tags ["batch_latency_99_9th" "onyx" task-name name peer-id-str]}))
                    (>!! ch (merge core {:service (format "[%s] max_batch_latency" task-name)
                                         :window "10s"
                                         :quantile 1.0
                                         :value (get latencies-vals 1.0)
                                         :metric :batch-latency
-                                        :tags ["batch_latency_max" "onyx" task-name name peer-id]})))
+                                        :tags ["batch_latency_max" "onyx" task-name name peer-id-str]})))
              (recur (mod (inc cycle-count) latency-period-secs)
                     (max 0 (- 1000 (- (System/currentTimeMillis) time-start))))))
          (catch InterruptedException e)


codahale metrics gauge implementation.

codahale metrics gauge implementation.

--- src/onyx/lifecycle/metrics/metrics.clj
+++ src/onyx/lifecycle/metrics/metrics.clj
@@ -8,7 +8,24 @@
             [metrics.timers :as t]
             [metrics.gauges :as g]
             [metrics.meters :as m])
-  (:import [java.util.concurrent TimeUnit]))
+  (:import [java.util.concurrent TimeUnit]
+           [java.util.concurrent.atomic AtomicLong]
+           [com.codahale.metrics Gauge]))
+
+(defprotocol IGauge
+  (setGauge [this value]))
+
+(deftype OGauge [value]
+  IGauge
+  (setGauge [this new-value]
+    (.set ^AtomicLong value ^long new-value))
+  Gauge
+  (getValue [this]
+    (.get ^AtomicLong value)))
+
+;(def v (->OGauge (AtomicLong.)))
+;(setGauge v 9933)
+;(.getValue v)

 (defn new-lifecycle-latency [reg tag lifecycle]
   (let [timer ^com.codahale.metrics.Timer (t/timer reg (into tag ["task-lifecycle" (name lifecycle)]))]


Use lowercase collections for compatibility with existing functions

--- src/pdok/featured_to_extracts/core.clj
+++ src/pdok/featured_to_extracts/core.clj
@@ -172,7 +172,7 @@
   (let [feature (t/from-json transit-line)]
     (merge (:attributes feature)
            {:_action (:action feature)
-            :_collection (:collection feature)
+            :_collection (.toLowerCase (:collection feature))
             :_id (:id feature)
             :_previous_version (:previous-version feature)
             :_version (:version feature)
@@ -185,7 +185,7 @@
   keys: feature-id,action,version,tiles,valid-from,old-version,old-tiles,old-valid-from,feature"
   (let [lines (line-seq (io/reader in-stream))
         version (first lines)
-        collection (:collection (t/from-json (second lines)))
+        collection (.toLowerCase (:collection (t/from-json (second lines))))
         ;drop collection info + header
         lines (drop 2 lines)]
     [version collection (map make-change-record lines)]))


Add test for update-thread-last-open

Add test for update-thread-last-open

--- test/braid/test/server/db.clj
+++ test/braid/test/server/db.clj
@@ -625,3 +625,30 @@
       (is (bot/bot-auth? (b2 :id) (b2 :token)))
       (is (not (bot/bot-auth? (b2 :id) "Foo")))
       (is (not (bot/bot-auth? (java.util.UUID/randomUUID) (b2 :token)))))))
+
+(deftest update-thread-open-at
+  (let [[{user-id :id} {group-id :id}]
+        (db/run-txns!
+          (concat
+            (user/create-user-txn {:id (db/uuid)
+                                   :email "foo@bar.com"
+                                   :password "foobar"
+                                   :avatar ""})
+            (group/create-group-txn {:id (db/uuid)
+                                     :name "group 1"})))
+        thread-id (db/uuid)]
+    (db/run-txns!
+      (message/create-message-txn {:id (db/uuid)
+                                   :thread-id thread-id
+                                   :group-id group-id
+                                   :user-id user-id
+                                   :created-at (java.util.Date.)
+                                   :content "foobar"
+                                   :mentioned-user-ids [user-id]
+                                   :mentioned-tag-ids []}))
+    (testing  "can update when a thread was last opened by user"
+      (let [thread (thread/thread-by-id thread-id)
+            old-open (:last-open-at (thread/thread-add-last-open-at thread user-id))
+            _ (thread/update-thread-last-open! thread-id user-id)
+            new-open (:last-open-at (thread/thread-add-last-open-at thread user-id))]
+        (is (< old-open new-open))))))


Fix tests for emulation of logistic module

--- test/cortex/function_test.clj
+++ test/cortex/function_test.clj
@@ -1,4 +1,4 @@
-(ns cortex.module-test
+(ns cortex.function-test
   "Tests for behaviour of mathematical function modules"
   (:use [clojure.test])
   (:use [cortex.core])
@@ -31,10 +31,17 @@
 (deftest test-function
   (testing "emulated logistic function"
     (let [m (layers/logistic [3])
-          m2 (layers/function m/logistic (fn [input output] (m/mul output (m/sub 1.0 output))))]
+          m2 (layers/function m/logistic
+                              (fn [input output-gradient]
+                                (let [output (m/logistic input)]
+                                  (m/mul output-gradient (m/mul output (m/sub 1.0 output))))))]
       (let [v [-10 1 10]]
         (is (m/equals (calc-output m v) (calc-output m2 v))))
-      (let [input [-10 1 10]
+      (let [train (fn [m input target]
+                    (let [m (forward m input)
+                          d (m/sub (output m) target)]
+                      (backward m input d)))
+            input [-10 1 10]
             target [0.1 0.5 0.6]
             m (train m input target)
             m2 (train m2 input target)]


Add positive test for multi-arity featurec

--- test/desdemona/query_test.clj
+++ test/desdemona/query_test.clj
@@ -199,7 +199,11 @@
   (testing "multi-arity featurec with literal"
     (are [query results] (= results (q/run-dsl-query 10 query events))
       '(= (:ip x) (:ip y) "1.2.3.4")
-      [])))
+      []
+
+      '(= (:ip x) (:ip y) "10.0.0.1")
+      [[{:ip "10.0.0.1"}
+        {:ip "10.0.0.1"}]])))

 (deftest run-logic-query-tests
   (are [query results] (= results (#'q/run-logic-query query events))


Add assertion message

Add assertion message

--- test/desdemona/query_test.clj
+++ test/desdemona/query_test.clj
@@ -100,7 +100,8 @@
                                 (= (:ip x) "10.0.0.1"))))))
   (testing "multiple literals"
     (is (thrown? IllegalArgumentException
-                 (#'q/dsl->logic '(= (:ip x) "10.0.0.1" "10.0.0.1"))))
+                 (#'q/dsl->logic '(= (:ip x) "10.0.0.1" "10.0.0.1")))
+        "repeated but consistent literal")
     (is (thrown? IllegalArgumentException
                  (#'q/dsl->logic '(= (:ip x) "10.0.0.1" "10.0.0.2")))))
   (testing "multiple terms unified with a literal"


mapc tests

mapc tests

--- test/grafter/tabular_test.clj
+++ test/grafter/tabular_test.clj
@@ -194,6 +194,63 @@
       (is (= (make-dataset [[1] [2]]) (drop-rows dataset 1)))
       (is (= (make-dataset [[2]]) (drop-rows dataset 2)))
       (is (= (make-dataset []) (drop-rows dataset 1000))))))
+
+(deftest grep-test
+  (let [dataset (make-dataset [["one" "two" "bar"]
+                               ["foo" "bar" "b2az"]
+                               ["foo" "blee" "bl3ah"]])
+
+        expected-dataset (make-dataset [["foo" "bar" "b2az"]
+                                        ["foo" "blee" "bl3ah"]])
+
+        expected-dataset-2 (make-dataset [["one" "two" "bar"]])]
+
+    (testing "grep"
+      (testing "with a function"
+        (testing "receives a single cell as its argument"
+          (grep dataset (fn [cell]
+                          (is (= String (class cell))))))
+
+        (is (= expected-dataset
+               (grep dataset (fn [cell]
+                               (= cell "foo")))))
+
+        (is (= expected-dataset
+               (grep dataset (fn [cell]
+                               (.startsWith cell "f")))))
+
+        (is (= expected-dataset-2
+               (grep dataset (fn [cell]
+                               (= cell "bar")) ["C"]))))
+      (testing "with a string"
+        (is (= expected-dataset
+               (grep dataset "fo"))))
+      (testing "with a regex"
+        (is (= expected-dataset
+               (grep ds #"\d")))))))
+
+(deftest mapc-test
+  (let [dataset (make-dataset [[1 2 "foo" 4]
+                               [5 6 "bar" 8]
+                               [9 10 "baz" 12]])
+        fs {"A" str, "B" inc, "C" identity, "D" inc}
+        fs-incomplete {"A" str, "B" inc, "D" inc}
+        fs-vec [str inc identity inc]
+        expected-dataset (make-dataset [["1" 3 "foo" 5]
+                                        ["5" 7 "bar" 9]
+                                        ["9" 11 "baz" 13]])]
+    (testing "mapc with a hashmap"
+      (testing "complete hashmap"
+        (is (= expected-dataset
+               (mapc dataset fs))))
+      (testing "incomplete hashmap"
+        (is (= expected-dataset
+               (mapc dataset fs-incomplete)))
+        (is (= dataset
+               (mapc dataset {})))))
+    (testing "mapc with a vector"
+      (is (= expected-dataset
+             (mapc dataset fs-vec))))))

 (deftest swap-test
   (let [ordered-ds (make-dataset
@@ -220,41 +277,6 @@
       (is (thrown? java.lang.Exception
              (swap ordered-ds "A" "B" "C"))))))

-(deftest grep-test
-  (let [dataset (make-dataset [["one" "two" "bar"]
-                               ["foo" "bar" "b2az"]
-                               ["foo" "blee" "bl3ah"]])
-
-        expected-dataset (make-dataset [["foo" "bar" "b2az"]
-                                        ["foo" "blee" "bl3ah"]])
-
-        expected-dataset-2 (make-dataset [["one" "two" "bar"]])]
-
-    (testing "grep"
-      (testing "with a function"
-        (testing "receives a single cell as its argument"
-          (grep dataset (fn [cell]
-                          (is (= String (class cell))))))
-
-        (is (= expected-dataset
-               (grep dataset (fn [cell]
-                               (= cell "foo")))))
-
-        (is (= expected-dataset
-               (grep dataset (fn [cell]
-                               (.startsWith cell "f")))))
-
-        (is (= expected-dataset-2
-               (grep dataset (fn [cell]
-                               (= cell "bar")) ["C"]))))
-      (testing "with a string"
-        (is (= expected-dataset
-               (grep dataset "fo"))))
-      (testing "with a regex"
-        (is (= expected-dataset
-               (grep ds #"\d")))))))
-
-
 (comment

 (deftest build-lookup-table-test


Cleanup

Cleanup

--- test/icecap/e2e/http_test.clj
+++ test/icecap/e2e/http_test.clj
@@ -1,6 +1,6 @@
 (ns icecap.e2e.http-test
   "End-to-end tests for making HTTP requests."
-  (:require [icecap.e2e.common :refer :all]
+  (:require [icecap.e2e.common :as e2e]
             [clojure.test :refer :all]
             [aleph.http :as http]
             [taoensso.timbre :refer [info spy]]))
@@ -40,8 +40,7 @@
   [f]
   (let [server (http/start-server handler {:port http-server-port})]
     (with-redefs [http-server server]
-      (try (f)
-           (finally (.close http-server))))))
+      (try (f) (finally (.close http-server))))))

 (defn ^:private store-reqs-fixture
   "A fixture that resets the request store."
@@ -49,16 +48,16 @@
   (with-redefs [recvd-reqs (atom [])]
     (f)))

-(use-fixtures :once icecap-fixture http-server-fixture)
+(use-fixtures :once e2e/icecap-fixture http-server-fixture)
 (use-fixtures :each store-reqs-fixture)

 (deftest http-tests
   (let [plan {:type :http
               :method :GET
               :url (str http-server-base-url "/test/example")}
-        create-result (spy @(create-cap plan))
-        cap-url (:cap (get-body create-result))
-        exercise-result @(execute-cap cap-url)]
+        create-result @(e2e/create-cap plan)
+        cap-url (:cap (spy (e2e/get-body create-result)))
+        exercise-result @(e2e/execute-cap cap-url)]
     ;; Was the cap successfully created?
     (is (= (:status create-result) 201))
     (is (some? cap-url))
@@ -80,6 +79,6 @@
              request)))))

 (deftest invalid-http-step-tests
-  (are [step expected] (= {:error expected} (get-body @(create-cap step)))
+  (are [step expected] (= {:error expected} (e2e/get-body @(e2e/create-cap step)))
     {:type :http} '{:url missing-required-key
                     :method missing-required-key}))


Adds unit test for change keys.

Adds unit test for change keys.

--- test/loudmoauth/util_test.clj
+++ test/loudmoauth/util_test.clj
@@ -5,6 +5,9 @@
             [clojure.core.async :as a]))

 ;(deftest test-change-keys)
+(deftest test-change-keys
+  (testing "Change hyphen to underscore for keys in map."
+    (is (= {:test_key_1 "1" :test_key_2 "2"}) (change-keys {:test-key-1 "1" :test_key_2 "2"}))))

 (deftest test-string-to-base64-string
   (testing "Conversion from normal string to base64 encoded string."


Namespace Doc string

Namespace Doc string

--- src/rdf-common/grafter/rdf/formats.clj
+++ src/rdf-common/grafter/rdf/formats.clj
@@ -1,4 +1,5 @@
 (ns grafter.rdf.formats
+  "Symbols used to specify different Linked Data Serializations."
   (:import [org.openrdf.rio RDFFormat]))

 (def rdf-xml RDFFormat/RDFXML)


Message queue: add support for worker loop monitor fn (for logging, etc.)

--- src/taoensso/carmine/message_queue.clj
+++ src/taoensso/carmine/message_queue.clj
@@ -192,7 +192,7 @@
     (when-not @running?
       (reset! running? true)
       (future
-        (let [{:keys [handler throttle-ms eoq-backoff-ms]} opts
+        (let [{:keys [handler monitor throttle-ms eoq-backoff-ms]} opts
               qk    (partial qkey qname)
               done  (fn [mid] (wcar conn (car/sadd (qk "recently-done") mid)))
               retry (fn [mid & [backoff-ms]]
@@ -209,15 +209,14 @@
             (try
               (let [[ndruns mid-circle-len] (wcar conn (car/get  (qk "ndry-runs"))
                                                        (car/llen (qk "mid-circle")))
-                    ndruns (or ndruns 0)
+                    ndruns (or (car/as-long ndruns) 0)
                     eoq-backoff-ms* (if (fn? eoq-backoff-ms)
                                       (eoq-backoff-ms (inc ndruns))
                                       eoq-backoff-ms)
                     opts* (assoc opts :eoq-backoff-ms eoq-backoff-ms*)]

-                ;; TODO For optional logging fn
-                ;; {:ndry-runs      ndruns
-                ;;  :mid-circle-len mid-circle-len}
+                (when monitor (monitor {:mid-circle-len mid-circle-len
+                                        :ndry-runs      ndruns}))

                 (when-let [[mid mcontent attempt :as poll-reply]
                            (wcar conn (dequeue qname opts*))]
@@ -251,6 +250,8 @@
                      or returns {:status     <#{:success :error :retry}>
                                  :throwable  <Throwable>
                                  :backoff-ms <retry-backoff-ms}.
+   :monitor        - (fn [{:keys [mid-circle-len ndry-runs]}]) called on each
+                     worker loop iteration. Useful for queue monitoring/logging.
    :lock-ms        - Max time handler may keep a message before handler
                      considered fatally stalled and message re-queued. Must be
                      sufficiently high to prevent double handling.
@@ -258,7 +259,8 @@
                      Can be a (fn [ndry-runs]) => ms. Sleep synchronized for all
                      queue workers.
    :throttle-ms    - Thread sleep period between each poll."
-  [conn qname & [{:keys [handler lock-ms eoq-backoff-ms throttle-ms auto-start?]
+  [conn qname & [{:keys [handler monitor lock-ms eoq-backoff-ms throttle-ms
+                         auto-start?]
                   :or   {handler (fn [{:keys [message attempt]}]
                                    (timbre/info qname message attempt)
                                    {:status :success})
@@ -268,6 +270,7 @@
                          auto-start?    true}}]]
   (let [w (->Worker conn qname (atom false)
                     {:handler        handler
+                     :monitor        monitor
                      :lock-ms        lock-ms
                      :eoq-backoff-ms eoq-backoff-ms
                      :throttle-ms    throttle-ms})]


Further progress with Hello World tests

Further progress with Hello World tests

--- test/yada/helloworld_test.clj
+++ test/yada/helloworld_test.clj
@@ -68,7 +68,24 @@
       [:headers "allow" parse-csv set] := #{"OPTIONS" "GET" "HEAD"}
       :body := nil)))

-;; TODO: mutable-hello
+(deftest atom-options-test
+  (let [resource (hello/hello-atom)]
+    (given @(resource (request :options "/"))
+      :status := 200
+      [:headers keys set] := #{"allow"}
+      [:headers validate-headers?] := []
+      [:headers "allow" parse-csv set] := #{"OPTIONS" "GET" "HEAD" "PUT" "POST" "DELETE"}
+      :body := nil)))
+
+#_(deftest put-test
+  (let [resource (hello/hello-atom)]
+    (given @(resource (merge (request :put "/")
+                             {:body "Hello Dolly!"}))
+      ;; What should a PUT return in this case?
+      :status := 204
+      [:headers keys set] := #{}
+      [:headers validate-headers?] := []
+      :body := nil)))

 ;; TODO: head request


DB interface test: insert-word-into-dictionary

--- test/zg/db_interface_test.clj
+++ test/zg/db_interface_test.clj
@@ -137,3 +137,31 @@
             "b" :blacklist
             "o" :other)))

+(deftest test-insert-word-into-dictionary
+    "Checking the function insert-word-into-dictionary."
+    (testing "the function insert-word-into-dictionary."
+        (with-redefs [jdbc/insert! (fn [db-spec table-name data-entry] table-name)]
+            (are [x y] (= x y)
+                :dictionary (insert-word-into-dictionary "word"           "description" "user-name"           :whitelist "2016-01-01")
+                :dictionary (insert-word-into-dictionary "word"           "description" "user-name"           :blacklist "2016-01-01")
+                :dictionary (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                :dictionary (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                :dictionary (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")
+                :dictionary (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")))
+        (with-redefs [jdbc/insert! (fn [db-spec table-name data-entry] (get data-entry :word))]
+            (are [x y] (= x y)
+                "word"           (insert-word-into-dictionary "word"           "description" "user-name"           :whitelist "2016-01-01")
+                "word"           (insert-word-into-dictionary "word"           "description" "user-name"           :blacklist "2016-01-01")
+                "different word" (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                "different word" (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                "different word" (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")
+                "different word" (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")))
+        (with-redefs [jdbc/insert! (fn [db-spec table-name data-entry] (get data-entry :dictionary))]
+            (are [x y] (= x y)
+                "w" (insert-word-into-dictionary "word"           "description" "user-name"           :whitelist "2016-01-01")
+                "b" (insert-word-into-dictionary "word"           "description" "user-name"           :blacklist "2016-01-01")
+                "b" (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                "b" (insert-word-into-dictionary "different word" "description" "user-name"           :blacklist "2016-01-01")
+                "b" (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")
+                "b" (insert-word-into-dictionary "different word" "description" "different user-name" :blacklist "2016-01-01")))))
+


Use fa-file-text for projects (it looks better than fa-code)

--- frontend/components/navbar.cljs
+++ frontend/components/navbar.cljs
@@ -108,7 +108,7 @@
          [:i.fa.fa-clock-o]
          "Recent builds"]
         [:a.menu-item {:href "/admin/projects"}
-         [:i.fa.fa-code]
+         [:i.fa.fa-file-text]
          "Projects"]
         [:a.menu-item
          {:on-click #(put! controls-ch [:intercom-user-inspected])}


Update for CLJS-1313 testing

Update for CLJS-1313 testing

--- node_repl.clj
+++ node_repl.clj
@@ -1,21 +1,11 @@
 (require 'cljs.repl)
-(require 'cljs.build.api)
 (require 'cljs.repl.node)

-(def foreign-libs [{:file "libs/greeting.js"
-       :provides ["greeting"]
-       :module-type :commonjs}
-      {:file "libs/german.js"
-       :provides ["german"]
-       :module-type :commonjs}])
-
-(cljs.build.api/build "src"
-  {:main 'foo.bar
-   :output-to "out/main.js"
-   :verbose true
-   :foreign-libs foreign-libs})
-
 (cljs.repl/repl (cljs.repl.node/repl-env)
-  :watch "src"
   :output-dir "out"
-  :foreign-libs foreign-libs)
+  :foreign-libs [{:file "libs/greeting.js"
+                  :provides ["greeting"]
+                  :module-type :commonjs}
+                 {:file "libs/german.js"
+                  :provides ["german"]
+                  :module-type :commonjs}])


CMR-1387: Fix order of facet response to be more readable

--- search-app/src/cmr/search/services/query_execution/facets_results_feature.clj
+++ search-app/src/cmr/search/services/query_execution/facets_results_feature.clj
@@ -102,11 +102,10 @@
           value-counts (for [bucket (get-in bucket-map [field :buckets])
                              :let [sub-facets (parse-hierarchical-bucket (rest field-hierarchy)
                                                                          bucket)]]
-                         (merge {:value (:key bucket)
-                                 :count (get-in bucket [:coll-count :doc_count]
-                                                (:doc_count bucket))}
-                                (when-not (= sub-facets empty-response)
-                                  sub-facets)))]
+                         (merge (when-not (= sub-facets empty-response)
+                                  sub-facets)
+                                {:count (get-in bucket [:coll-count :doc_count] (:doc_count bucket))
+                                 :value (:key bucket)}))]
       (if (seq value-counts)
         {:subfields [field]
          field value-counts}


Correct misnumbered 403 error codes for swagger (#1363)

--- scheduler/src/cook/rest/api.clj
+++ scheduler/src/cook/rest/api.clj
@@ -2926,7 +2926,7 @@
                    :responses {200 {:schema [JobResponseDeprecated]
                                     :description "The jobs and their instances were returned."}
                                400 {:description "Non-UUID values were passed as jobs."}
-                               403 {:description "The supplied UUIDs don't correspond to valid jobs."}}
+                               404 {:description "The supplied UUIDs don't correspond to valid jobs."}}
                    :handler (read-jobs-handler-deprecated conn is-authorized-fn)}
              :post {:summary "Schedules one or more jobs (deprecated)."
                     :parameters {:body-params RawSchedulerRequestDeprecated}
@@ -2937,7 +2937,7 @@
              :delete {:summary "Cancels jobs, halting execution when possible."
                       :responses {204 {:description "The jobs have been marked for termination."}
                                   400 {:description "Non-UUID values were passed as jobs."}
-                                  403 {:description "The supplied UUIDs don't correspond to valid jobs."}}
+                                  404 {:description "The supplied UUIDs don't correspond to valid jobs."}}
                       :parameters {:query-params JobOrInstanceIds}
                       :handler (destroy-jobs-handler conn is-authorized-fn)}}))

@@ -2949,7 +2949,7 @@
                    :responses {200 {:schema JobResponse
                                     :description "The job was returned."}
                                400 {:description "A non-UUID value was passed."}
-                               403 {:description "The supplied UUID doesn't correspond to a valid job."}}
+                               404 {:description "The supplied UUID doesn't correspond to a valid job."}}
                    :handler (read-jobs-handler-single conn is-authorized-fn)}}))

         (c-api/context
@@ -2960,7 +2960,7 @@
                    :responses {200 {:schema [JobResponse]
                                     :description "The jobs were returned."}
                                400 {:description "Non-UUID values were passed."}
-                               403 {:description "The supplied UUIDs don't correspond to valid jobs."}}
+                               404 {:description "The supplied UUIDs don't correspond to valid jobs."}}
                    :handler (read-jobs-handler-multiple conn is-authorized-fn)}
              :post {:summary "Schedules one or more jobs."
                     :parameters {:body-params JobSubmissionRequest}
@@ -2985,7 +2985,7 @@
                    :responses {200 {:schema InstanceResponse
                                     :description "The job instance was returned."}
                                400 {:description "A non-UUID value was passed."}
-                               403 {:description "The supplied UUID doesn't correspond to a valid job instance."}}
+                               404 {:description "The supplied UUID doesn't correspond to a valid job instance."}}
                    :handler (read-instances-handler-single conn is-authorized-fn)}}))

         (c-api/context
@@ -2996,7 +2996,7 @@
                    :responses {200 {:schema [InstanceResponse]
                                     :description "The job instances were returned."}
                                400 {:description "Non-UUID values were passed."}
-                               403 {:description "The supplied UUIDs don't correspond to valid job instances."}}
+                               404 {:description "The supplied UUIDs don't correspond to valid job instances."}}
                    :handler (read-instances-handler-multiple conn is-authorized-fn)}}))

         (c-api/context
@@ -3005,8 +3005,8 @@
             {:produces ["application/json"],
              :responses {200 {:schema UserLimitsResponse
                               :description "User's share found"}
-                         401 {:description "Not authorized to read shares."}
-                         403 {:description "Invalid request format."}}
+                         400 {:description "Invalid request format."}
+                         401 {:description "Not authorized to read shares."}}
              :get {:summary "Read a user's share"
                    :parameters {:query-params UserParam}
                    :handler (read-limit-handler :share share/get-share conn is-authorized-fn)}
@@ -3025,8 +3025,8 @@
             {:produces ["application/json"],
              :responses {200 {:schema UserLimitsResponse
                               :description "User's quota found"}
-                         401 {:description "Not authorized to read quota."}
-                         403 {:description "Invalid request format."}}
+                         400 {:description "Invalid request format."}
+                         401 {:description "Not authorized to read quota."}}
              :get {:summary "Read a user's quota"
                    :parameters {:query-params UserParam}
                    :handler (read-limit-handler :quota quota/get-quota conn is-authorized-fn)}
@@ -3045,8 +3045,8 @@
             {:produces ["application/json"],
              :responses {200 {:schema UserUsageResponse
                               :description "User's usage calculated."}
-                         401 {:description "Not authorized to read usage."}
-                         403 {:description "Invalid request format."}}
+                         400 {:description "Invalid request format."}
+                         401 {:description "Not authorized to read usage."}}
              :get {:summary "Query a user's current resource usage."
                    :parameters {:query-params UserUsageParams}
                    :handler (read-usage-handler conn is-authorized-fn)}}))
@@ -3089,14 +3089,14 @@
                    :responses {200 {:schema [GroupResponse]
                                     :description "The groups were returned."}
                                400 {:description "Non-UUID values were passed."}
-                               403 {:description "The supplied UUIDs don't correspond to valid groups."}}
+                               404 {:description "The supplied UUIDs don't correspond to valid groups."}}
                    :handler (groups-action-handler conn task-constraints is-authorized-fn)}
              :delete
              {:summary "Kill all jobs within a set of groups"
               :parameters {:query-params KillGroupsParams}
               :responses {204 {:description "The groups' jobs have been marked for termination."}
                           400 {:description "Non-UUID values were passed."}
-                          403 {:description "The supplied UUIDs don't correspond to valid groups."}}
+                          404 {:description "The supplied UUIDs don't correspond to valid groups."}}
               :handler (groups-action-handler conn task-constraints is-authorized-fn)}}))

         (c-api/context
@@ -3132,8 +3132,8 @@
             {:produces ["application/json"],
              :responses {200 {:schema TaskStatsResponse
                               :description "Task stats calculated."}
-                         401 {:description "Not authorized to read stats."}
-                         403 {:description "Invalid request format."}}
+                         400 {:description "Invalid request format."}
+                         401 {:description "Not authorized to read stats."}}
              :get {:summary "Query statistics for instances started in a time range."
                    :parameters {:query-params TaskStatsParams}
                    :handler (task-stats-handler conn is-authorized-fn)}}))


implement monitor removal

implement monitor removal

--- test/clj_money/models/reports_test.clj
+++ test/clj_money/models/reports_test.clj
@@ -565,9 +565,10 @@
         groceries (-> context :accounts (get 6))

         ; half-way through january
-        actual (reports/monitor storage-spec
+        actual (-> (reports/monitor storage-spec
                                 groceries
                                 {:as-of (t/local-date 2016 1 15)})
+                   (dissoc :account))
         expected {:caption "Groceries"
                   :period {:total-budget 450M
                            :prorated-budget 217.74M


added hansen field

added hansen field


Post an update every five seconds.

Post an update every five seconds.

--- metlog-agent/src/metlog_agent/core.clj
+++ metlog-agent/src/metlog_agent/core.clj
@@ -17,21 +17,19 @@
   (locking sensor-result-queue
     (.add sensor-result-queue (poll-sensor))))

-(def vault-update-queue (java.util.concurrent.LinkedBlockingQueue.))
-
 (defn update-vault []
   (log/info "update-vault")
   (let [ snapshot (java.util.concurrent.LinkedBlockingQueue.) ]
     (locking sensor-result-queue
       (.drainTo sensor-result-queue snapshot))
     (doseq [ item (seq snapshot)]
-      (println item))))
+      (log/info item))))

 (defn -main
   "I don't do a whole lot ... yet."
   [& args]
   (at-at/every 1000 (exception-barrier poll-sensors) my-pool)
-  (at-at/every 1000 (exception-barrier update-vault) my-pool)
+  (at-at/every 5000 (exception-barrier update-vault) my-pool)
   (println "Running."))


Fix compilation error

Fix compilation error

--- src/io/cyanite/store.clj
+++ src/io/cyanite/store.clj
@@ -139,7 +139,7 @@
 (defn cassandra-metric-store
   "Connect to cassandra and start a path fetching thread.
    The interval is fixed for now, at 1minute"
-  [{:keys [keyspace cluster hints repfactor]
+  [{:keys [keyspace cluster hints repfactor chan_size batch_size]
     :or   {hints {:replication {:class "SimpleStrategy"
                                 :replication_factor (or repfactor 3)}}
            chan_size 10000


CMR-1383 Adds call to validate-provider for provider updates

--- metadata-db-app/src/cmr/metadata_db/services/provider_service.clj
+++ metadata-db-app/src/cmr/metadata_db/services/provider_service.clj
@@ -78,6 +78,7 @@
   "Updates a provider."
   [context {:keys [provider-id] :as provider}]
   (info "Updating provider [" provider-id "]")
+  (validate-provider provider)
   (let [db (mdb-util/context->db context)
         result (providers/update-provider db provider)
         error-code (:error result)]


add failing test for checker called twice

--- test/midje/checking/t_core.clj
+++ test/midje/checking/t_core.clj
@@ -97,3 +97,18 @@
     (evaluate-checking-function (just 1) [2]) => [false extra])
   (let [error (Error.)]
     (evaluate-checking-function (fn [actual] (throw error)) anything) => [false {:thrown error}]))
+
+(def fail-call-count (atom 0))
+(def succeed-call-count (atom 0))
+
+(facts "Test checking function call counts"
+  (reset! fail-call-count 0)
+  (reset! succeed-call-count 0)
+  (silent-fact "Register call count of failing checking function"
+    nil => (fn [actual] (do (swap! fail-call-count inc) false)))
+  (note-that (fact-actual nil))
+  (fact "Register call count for successful checking function"
+    nil => (fn [actual] (do (swap! succeed-call-count inc) true)))
+  (fact "Checking functions are only called once, independent of success or failure"
+    @fail-call-count => 1
+    @succeed-call-count => 1))


Fixed context/out! arity

Fixed context/out! arity

--- src/lt/objs/context.cljs
+++ src/lt/objs/context.cljs
@@ -20,14 +20,14 @@
 (declare ctx-obj)

 (defn out!
-  ([] (out! nil))
   ([ctxs]
    (let [ctxs (if (coll? ctxs)
                 ctxs
                 [ctxs])]
      (swap! contexts #(apply disj % ctxs))
      (swap! ctx->obj #(apply dissoc % ctxs))
-    (object/raise ctx-obj :log!))))
+    (object/raise ctx-obj :log!)))
+   ([ctxs _] (out! ctxs)))

 (defn in! [ctxs & [obj]]
   (let [ctxs (if (coll? ctxs)


sort by command

sort by command

--- src/ophion/aggregate.clj
+++ src/ophion/aggregate.clj
@@ -122,8 +122,8 @@
    :desc -1})

 (defn qount
-  [label]
-  [{:$count label}])
+  []
+  [{:$count "count"}])

 (defn dedup
   []
@@ -166,6 +166,15 @@
 (defn element-at
   [label]
   {:$arrayElemAt [(dollar label) 0]})
+
+(defn sort-order
+  [direction]
+  (if (= "desc" (name direction)) -1 1))
+
+(defn sortBy
+  [field direction]
+  (let [order (sort-order direction)]
+    [{:$sort {(name field) order}}]))

 (declare translate)

@@ -235,6 +244,7 @@
    :order order
    :offset offset
    :count qount
+   :sort sortBy
    :groupCount group-count})

 (defn apply-step


fixing weird merge issue...

fixing weird merge issue...

--- src/overtone/sc/node.clj
+++ src/overtone/sc/node.clj
@@ -144,14 +144,8 @@
 (defn group-free
   "Free synth groups, releasing their resources."
   [& group-ids]
-<<<<<<< HEAD
-  {:pre [(connected?)]}
-  (apply node-free group-ids)
-  :free)
-=======
   {:pre [(server-connected?)]}
   (apply node-free group-ids))
->>>>>>> 10a6172a2f2dfc06defee0e1b8a7fd68e73cf924

 (defn node-run
   "Start a stopped synth node."


Describe the stream-id uniqueness constraints

It was unclear that stream ids must be stringifiable.

--- src/rill/event_store.clj
+++ src/rill/event_store.clj
@@ -20,6 +20,9 @@
    "Returns a sequence of the events in stream `stream-id', starting from the
     given cursor. If there are no more events at that cursor, wait for
     `wait-for-seconds' to see if anything new shows up.
+
+    `stream-id' may be stringified (using str) for comparison, so stream ids
+    that stringify to the same value are assumed to describe the same stream.

     The initial cursor for retrieving a complete event stream is -1. See also
     `rill.event-store/retrieve-events'. Valid cursors for retrieval from a


Add current version to download page

Add current version to download page

--- src/sinon_docs/pages.clj
+++ src/sinon_docs/pages.clj
@@ -37,6 +37,7 @@
    context
    "Downloads"
    (-> (slurp "resources/partials/download.html")
+       (insert-download-button (sinon/current-release))
        (insert-historic-download-links (sinon/historic-releases)))))

 (defn qunit-page [context]


remove dead code

remove dead code

--- src/spec_tools/core.cljc
+++ src/spec_tools/core.cljc
@@ -50,11 +50,6 @@
       (ex-info
         (str "can't coerce to spec: " name-or-spec)
         {:name-or-spec name-or-spec}))))
-
-(defn ^:skip-wiki set-of [value]
-  (s/coll-of
-    value
-    :into #{}))

 (defn ^:skip-wiki serialize
   "Writes specs into a string that can be read by the reader.


support :prepend in :node-library builds

fixes #414

--- src/main/shadow/build/targets/node_library.clj
+++ src/main/shadow/build/targets/node_library.clj
@@ -111,9 +111,12 @@
             (str/split #"//CLJS-HERE"))

         prepend
-        (cond-> prepend
-          umd-root-name
-          (str/replace #"root.returnExports" (str "root." umd-root-name)))]
+        (-> (when-let [prep (:prepend config)]
+              (str (str/trim prep) "\n"))
+            (str prepend)
+            (cond->
+              umd-root-name
+              (str/replace #"root.returnExports" (str "root." umd-root-name))))]

     (-> state
         (assoc :node-config node-config)


add the :join-tables clause to expanded query dict

--- src/metabase/driver/query_processor/expand.clj
+++ src/metabase/driver/query_processor/expand.clj
@@ -44,6 +44,7 @@
             [metabase.db :refer [sel]]
             (metabase.models [database :refer [Database]]
                              [field :as field]
+                             [foreign-key :refer [ForeignKey]]
                              [table :refer [Table]])
             [metabase.util :as u])
   (:import (clojure.lang Keyword)))
@@ -118,15 +119,39 @@
   [{database-id :database, :as expanded-query-dict}]
   (assoc expanded-query-dict :database (sel :one :fields [Database :name :id :engine :details] :id database-id)))

+(defn- join-tables-fetch-field-info
+  "Fetch info for PK/FK `Fields` for the JOIN-TABLES referenced in a Query."
+  [source-table-id join-tables]
+  (let [ ;; Build a map of source table FK field IDs -> field names
+        fk-field-id->field-name      (sel :many :id->field [field/Field :name], :table_id source-table-id, :special_type "fk")
+
+        ;; Build a map of join table PK field IDs -> source table FK field IDs
+        pk-field-id->fk-field-id     (sel :many :field->field [ForeignKey :destination_id :origin_id],
+                                          :origin_id [in (set (keys fk-field-id->field-name))])
+
+        ;; Build a map of join table ID -> PK field info
+        join-table-id->pk-field      (let [pk-fields (sel :many :fields [field/Field :id :table_id :name], :id [in (set (keys pk-field-id->fk-field-id))])]
+                                       (zipmap (map :table_id pk-fields) pk-fields))]
+
+    ;; Now build the :join-tables clause
+    (vec (for [{table-id :id, :as table} join-tables]
+           (let [pk-field      (join-table-id->pk-field table-id)
+                 fk-field-id   (pk-field-id->fk-field-id (:id pk-field))
+                 fk-field-name (fk-field-id->field-name fk-field-id)]
+             (assoc table
+                    :pk-field     pk-field
+                    :source-field {:id         fk-field-id
+                                   :name       fk-field-name}))))))
+
 (defn- resolve-tables
   "Resolve the `Tables` in an EXPANDED-QUERY-DICT."
   [{{source-table-id :source-table} :query, database-id :database, :as expanded-query-dict} table-ids]
-  (let [table-id->table (sel :many :id->fields [Table :name :id] :id [in table-ids])
-        join-tables     (vals (dissoc table-id->table source-table-id))]
-    (walk/postwalk #(resolve-table % table-id->table)
-                   (cond-> (assoc-in expanded-query-dict [:query :source-table] (table-id->table source-table-id))
-                     ;; If there are any Tables to join include them in a :join-tables clause
-                     join-tables (assoc-in [:query :join-tables] (vec join-tables))))))
+  (let [table-id->table              (sel :many :id->fields [Table :name :id] :id [in table-ids])
+        join-tables                  (vals (dissoc table-id->table source-table-id))]
+    (->> (assoc-in expanded-query-dict [:query :source-table] (table-id->table source-table-id))
+         (#(if-not join-tables %
+                   (assoc-in % [:query :join-tables] (join-tables-fetch-field-info source-table-id join-tables))))
+         (walk/postwalk #(resolve-table % table-id->table)))))

 ;; ## -------------------- Public Interface --------------------


Added missing spec for :sync-data key

Added missing spec for :sync-data key

--- src/status_im/specs.cljs
+++ src/status_im/specs.cljs
@@ -28,6 +28,7 @@
 ;NODE
 (s/def ::sync-listening-started (s/nilable boolean?))
 (s/def ::sync-state (s/nilable keyword?))
+(s/def ::sync-data (s/nilable map?))
 ;NETWORK
 (s/def ::network (s/nilable keyword?))                                  ;;network name :testnet

@@ -48,6 +49,7 @@
                            ::network-status
                            ::sync-listening-started
                            ::sync-state
+                           ::sync-data
                            ::network
                            :accounts/accounts
                            :accounts/account-creation?


Housekeeping: merge

Housekeeping: merge

--- src/taoensso/faraday.clj
+++ src/taoensso/faraday.clj
@@ -231,9 +231,10 @@
      :creation-date (.getCreationDateTime d)
      :item-count    (.getItemCount d)
      :size          (.getTableSizeBytes d)
-     :throughput    (as-map (.getProvisionedThroughput d))
-     :indexes       (as-map (.getLocalSecondaryIndexes d))
-     :gindexes (as-map (.getGlobalSecondaryIndexes d))
+     :throughput    (as-map (.getProvisionedThroughput  d))
+     :indexes       (as-map (.getLocalSecondaryIndexes  d)) ; DEPRECATED
+     :lsindexes     (as-map (.getLocalSecondaryIndexes  d))
+     :gsindexes     (as-map (.getGlobalSecondaryIndexes d))
      :status        (utils/un-enum (.getTableStatus d))
      :prim-keys
      (let [schema (as-map (.getKeySchema d))
@@ -263,10 +264,10 @@
   GlobalSecondaryIndexDescription
   (as-map [d] {:name       (keyword (.getIndexName d))
                :size       (.getIndexSizeBytes d)
-               :throughput (as-map (.getProvisionedThroughput d))
                :item-count (.getItemCount d)
                :key-schema (as-map (.getKeySchema d))
-               :projection (as-map (.getProjection d))})
+               :projection (as-map (.getProjection d))
+               :throughput (as-map (.getProvisionedThroughput d))})

   ProvisionedThroughputDescription
   (as-map [d] {:read                (.getReadCapacityUnits d)
@@ -332,11 +333,11 @@
     (.setWriteCapacityUnits (long write-units))))

 (defn- keydefs "[<name> <type>] defs -> [AttributeDefinition ...]"
-  [hash-keydef range-keydef indexes gindexes]
+  [hash-keydef range-keydef lsindexes gsindexes]
   (let [defs (->> (conj [] hash-keydef range-keydef)
-                  (concat (mapv :range-keydef indexes))
-                  (concat (mapv :hash-keydef gindexes))
-                  (concat (mapv :range-keydef gindexes))
+                  (concat (mapv :range-keydef lsindexes))
+                  (concat (mapv :hash-keydef  gsindexes))
+                  (concat (mapv :range-keydef gsindexes))
                   (filterv identity))]
     (mapv
      (fn [[key-name key-type :as def]]
@@ -348,17 +349,17 @@
          (.setAttributeType (utils/enum key-type))))
      defs)))

-(defn- local-indexes
+(defn- local-2nd-indexes
   "[{:name _ :range-keydef _ :projection _} ...] indexes -> [LocalSecondaryIndex ...]"
   [hash-keydef indexes]
-  (if indexes
+  (when indexes
     (mapv
       (fn [{index-name :name
-            :keys      [range-keydef projection]
-            :or        {projection :all}
-            :as        index}]
+           :keys      [range-keydef projection]
+           :or        {projection :all}
+           :as        index}]
         (assert (and index-name range-keydef projection)
-                (str "Malformed index: " index))
+                (str "Malformed local secondary index (LSI): " index))
         (doto (LocalSecondaryIndex.)
           (.setIndexName (name index-name))
           (.setKeySchema (key-schema hash-keydef range-keydef))
@@ -366,54 +367,64 @@
             (let [pr (Projection.)
                   ptype (if (coll?* projection) :include projection)]
               (.setProjectionType pr (utils/enum ptype))
-              (when (= ptype :include) (.setNonKeyAttributes pr (mapv name projection)))
+              (when (= ptype :include)
+                (.setNonKeyAttributes pr (mapv name projection)))
               pr))))
       indexes)))

-(defn- global-indexes
-  "[{:name _ :hash-keydef _ :range-keydef _ :throughput _ :projection _} ...] gindexes -> [GlobalSecondaryIndex ...]"
-  [gindexes]
-  (if gindexes
+(defn- global-2nd-indexes
+  "[{:name _ :hash-keydef _ :range-keydef _ :projection _ :throughput _} ...]
+  indexes -> [GlobalSecondaryIndex ...]"
+  [indexes]
+  (when indexes
     (mapv
       (fn [{index-name :name
-            :keys      [hash-keydef range-keydef throughput projection]
-            :or        {projection :all}
-            :as        index}]
+           :keys      [hash-keydef range-keydef throughput projection]
+           :or        {projection :all}
+           :as        index}]
         (assert (and index-name hash-keydef projection throughput)
-                (str "Malformed index: " index))
+                (str "Malformed global secondary index (GSI): " index))
         (doto (GlobalSecondaryIndex.)
           (.setIndexName (name index-name))
           (.setKeySchema (key-schema hash-keydef range-keydef))
-          (.setProvisionedThroughput (provisioned-throughput throughput))
           (.setProjection
             (let [pr (Projection.)
                   ptype (if (coll?* projection) :include projection)]
               (.setProjectionType pr (utils/enum ptype))
-              (when (= ptype :include) (.setNonKeyAttributes pr (mapv name projection)))
-              pr))))
-      gindexes)))
+              (when (= ptype :include)
+                (.setNonKeyAttributes pr (mapv name projection)))
+              pr))
+          (.setProvisionedThroughput (provisioned-throughput throughput))))
+      indexes)))

 (defn create-table
   "Creates a table with options:
     hash-keydef   - [<name> <#{:s :n :ss :ns :b :bs}>].
     :range-keydef - [<name> <#{:s :n :ss :ns :b :bs}>].
     :throughput   - {:read <units> :write <units>}.
-    :indexes      - [{:name _ :range-keydef _
+    :lsindexes    - [{:name _ :range-keydef _
                       :projection #{:all :keys-only [<attr> ...]}}].
+    :gsindexes    - [{:name _ :hash-keydef _ :range-keydef _
+                      :projection #{:all :keys-only [<attr> ...]}
+                      :throughput _}].
     :block?       - Block for table to actually be active?"
   [creds table-name hash-keydef
-   & [{:keys [range-keydef throughput indexes gindexes block?]
-       :or   {throughput {:read 1 :write 1}}}]]
-  (let [result
+   & [{:keys [range-keydef throughput lsindexes gsindexes block?]
+       :or   {throughput {:read 1 :write 1}} :as opts}]]
+  (let [lsindexes (or lsindexes (:indexes opts)) ; DEPRECATED
+        result
         (as-map
          (.createTable (db-client creds)
            (doto-cond [_ (CreateTableRequest.)]
              :always (.setTableName (name table-name))
              :always (.setKeySchema (key-schema hash-keydef range-keydef))
              :always (.setProvisionedThroughput (provisioned-throughput throughput))
-             :always (.setAttributeDefinitions  (keydefs hash-keydef range-keydef indexes gindexes))
-             :always (.setLocalSecondaryIndexes (local-indexes hash-keydef indexes))
-             gindexes (.setGlobalSecondaryIndexes (global-indexes gindexes)))))]
+             :always (.setAttributeDefinitions
+                      (keydefs hash-keydef range-keydef lsindexes gsindexes))
+             lsindexes (.setLocalSecondaryIndexes
+                        (local-2nd-indexes hash-keydef lsindexes))
+             gsindexes (.setGlobalSecondaryIndexes
+                        (global-2nd-indexes gsindexes)))))]
     (if-not block? result @(table-status-watch creds table-name :creating))))

 (comment (time (create-table mc "delete-me7" [:id :s] {:block? true})))
@@ -719,7 +730,7 @@
                      stitching.
     :return        - e/o #{:all-attributes :all-projected-attributes :count
                            [<attr> ...]}.
-    :index         - Name of a secondary index to query.
+    :index         - Name of a local or global secondary index to query.
     :order         - Index scaning order e/o #{:asc :desc}.
     :limit         - Max num >=1 of items to eval (≠ num of matching items).
                      Useful to prevent harmful sudden bursts of read activity.


FIX: explicitly init record-keeper

FIX: explicitly init record-keeper

--- ssclj/jar/src/main/clojure/com/sixsq/slipstream/ssclj/db/database_binding.clj
+++ ssclj/jar/src/main/clojure/com/sixsq/slipstream/ssclj/db/database_binding.clj
@@ -10,7 +10,9 @@
     [honeysql.core                                          :as hq]
     [com.sixsq.slipstream.ssclj.database.korma-helper       :as kh]

+    [com.sixsq.slipstream.ssclj.usage.record-keeper         :as rc]
     [com.sixsq.slipstream.ssclj.api.acl                     :as acl]
+
     [com.sixsq.slipstream.ssclj.db.binding                  :refer [Binding]]
     [com.sixsq.slipstream.ssclj.db.filesystem-binding-utils :refer [serialize deserialize]]
     [com.sixsq.slipstream.ssclj.database.ddl                :as ddl]
@@ -20,6 +22,7 @@
 (defn init-db
   []
   (acl/-init)
+  (rc/-init)
   (kh/korma-init)
   (ddl/create-table! "resources" (ddl/columns "id" "VARCHAR(100)" "data" "VARCHAR(10000)"))
   (defentity resources))


Added io.clj, old version (pre lib support)

--- rosado/io/io.clj
+++ rosado/io/io.clj
@@ -0,0 +1,50 @@
+;; simple io abstraction layer for clojure
+;; Roland Sadowski [szabla gmail com]
+
+(in-ns 'io)
+(clojure/refer 'clojure)
+
+(import '(java.io File FileReader FileWriter BufferedReader BufferedWriter
+				  FileInputStream FileOutputStream FileOutputStream
+				  BufferedInputStream BufferedOutputStream))
+
+(defn reader
+  "Returns a reader or input stream (buffered).
+  optional keywords: :unbuff [:ub] - for unbuffered input.
+  :bytes - for an InputStream instead of a Reader."
+  [#^String file-name & modes]
+  (cond
+   (nil? modes) (BufferedReader. (FileReader. file-name))
+   (some #{:bytes} modes) (if (some #{:unbuff :ub} modes)
+							(FileInputStream. file-name)
+							(BufferedInputStream. (FileInputStream. file-name)))
+   (some #{:unbuff :ub} modes) (FileReader. file-name)))
+
+(defn read-lines
+  "Returns a list of lines from seq."
+  [#^java.io.BufferedReader rdr]
+  (loop [lines [] line (.readLine rdr)]
+	(if (nil? line)
+	  lines
+	  (recur (conj lines line) (.readLine rdr)))))
+
+(defn writer
+  "Returns a writer or output stream (buffered).
+  optional keywords: :unbuff [:ub] - for unbuffered output.
+  :bytes - for an OutputStream instead of a Writer."
+  [#^String file-name & modes]
+  (cond
+   (nil? modes) (BufferedWriter. (FileWriter. file-name))
+   (some #{:bytes} modes) (if (some #{:unbuff :ub} modes)
+							(FileOutputStream. file-name)
+							(BufferedOutputStream. (FileOutputStream. file-name)))
+   (some #{:unbuff :ub} modes) (FileWriter. file-name)))
+
+(defn write-lines
+  "Writes a seq of lines to a writer. Appends a newline to ecah line."
+  [#^java.io.BufferedWriter a-writer lines]
+  (loop [lns lines]
+	(when lns
+	  (.write a-writer (first lns))
+	  (.newLine a-writer)
+	  (recur (rest lns)))))


Fixed typo in uniprot tremble protein labels

--- resources/rules/post_identifier_merge/bio_labels/uniprot_swissprot_protein_labels.clj
+++ resources/rules/post_identifier_merge/bio_labels/uniprot_swissprot_protein_labels.clj
@@ -18,7 +18,7 @@
          (?/protein_symbol_field_value rdf/type ccp/IAO_EXT_0000932) ; ccp:UniProt_protein_record__name_field_value
          (?/protein_symbol_field_value rdfs/label ?/protein_symbol))

-   ;; In this case no `:reify` entry is required, since all of the data required to form the new triples already exist
+   ;; In this case no :reify entry is required, since all of the data required to form the new triples already exist

    :options {:magic-prefixes [["franzOption_chunkProcessingAllowed" "franz:yes"]]}
    }


CMR-2466 Cleanup code for the group service message.

--- access-control-app/src/cmr/access_control/services/group_service_messages.clj
+++ access-control-app/src/cmr/access_control/services/group_service_messages.clj
@@ -9,9 +9,8 @@
   "Gets a group name from a access-group concept's metadata.
    This is because we lowercase the native-id so it does not match the actual group name."
   [concept]
-  (let [metadata (:metadata concept)
-        name (:name (edn/read-string metadata))]
-    name))
+  (-> concept :metadata edn/read-string :name))
+

 (defn group-already-exists
   [group concept]


CMR-3366: Removed the "NOT APPLICABLE" place holder in util.clj

--- umm-spec-lib/src/cmr/umm_spec/util.clj
+++ umm-spec-lib/src/cmr/umm_spec/util.clj
@@ -145,10 +145,6 @@
   "place holder string value for NOT PROVIDED string field"
   "NOT PROVIDED")

-(def NOT-APPLICABLE
-  "place holder string value for NOT APPLICABLE string field"
-  "NOT APPLICABLE")
-
 (def STRING
   "place holder string value for string field not in the enum list."
   "STRING")


Отключил лишние меседж боксы в транзите

Отключил лишние меседж боксы в транзите

--- radiolamp-6P3S/src/ixinfestor/net.cljs
+++ radiolamp-6P3S/src/ixinfestor/net.cljs
@@ -31,7 +31,7 @@

 (defn get-data [uri params success-fn & [error-fn complete-fn disable-progress-element?]]
-  (letfn [(print-error [s ee]
+  (letfn [(print-error [s ee show-alert]
             (println "ERROR IN " s "\n"
                      "LastErrorCode: " (.getLastErrorCode ee) "\n"
                      "Status: " (.getStatus ee) " - " (.getStatusText ee) "\n"
@@ -39,9 +39,10 @@
                      "---------------------------------------------------------\n"
                      (.getResponseText ee)
                      "\n---------------------------------------------------------\n\n")
-            (js/alert (str
-                       "ERROR IN [" s "]\n"
-                       "Status: " (.getStatus ee) " - " (.getStatusText ee))))
+            (when show-alert
+              (js/alert (str
+                         "ERROR IN [" s "]\n"
+                         "Status: " (.getStatus ee) " - " (.getStatusText ee)))))

           ;; (redirect-to-root-when-not-transit [ee]
           ;;   (let [ct (.getResponseHeader ee "Content-Type")]
@@ -71,12 +72,12 @@
                          (try
                            (let [response (t/read r rt)]
                              (make-reload-if-reload-client r)
-                             (print-error "ERROR: " ee)
-                             (js/alert (str "ОШИБКА ОБРАЩЕНИЯ К СЕРВЕРУ:\n" response))
+                             (print-error "ERROR: " ee false)
+                             ;;(js/alert (str "ОШИБКА ОБРАЩЕНИЯ К СЕРВЕРУ:\n" response))
                              (if error-fn (error-fn response)))
                            (catch js/Error e
                              (let [m (str "ERROR: " e)]
-                               (print-error "ERRON IN ERROR:" ee)
+                               (print-error "ERRON IN ERROR:" ee true)
                                (js/alert (str  "ERRON IN ERROR: " rt))))))))

       (events/listen req goog.net.EventType.COMPLETE
@@ -84,7 +85,7 @@
                        (let [ee (.-target e)]
                          ;;;(redirect-to-root-when-not-transit ee)
                          (when-not (.isSuccess ee)
-                           (print-error "COMPLETE" ee))
+                           (print-error "COMPLETE" ee true))

                          ;; Скрыть элемент прогресса выполнения
                          (when progress-element
@@ -109,7 +110,7 @@
                            (catch js/Error e
                              (let [m (str "Ошибка сохранения: " e)]
                                (println m)
-                               (print-error "ERRON IN REQUEST SUCCESS" ee)))))))
+                               (print-error "ERRON IN REQUEST SUCCESS" ee false)))))))
       (.send req
              uri
              "POST"


Add Project Checkout Keys Paths

Add Project Checkout Keys Paths

  * Add dangerzone routes

--- src-cljs/frontend/api/path.cljs
+++ src-cljs/frontend/api/path.cljs
@@ -9,6 +9,24 @@
     "github" (gstring/format
                "/api/v1/project/%s/%s/settings"
                org repo)))
+
+(defn project-checkout-keys [vcs-type repo-name]
+  (case vcs-type
+    "bitbucket" (gstring/format
+                  "/api/dangerzone/project/%s/%s/checkout-key"
+                  vcs-type repo-name)
+    "github" (gstring/format
+               "/api1/v1/project/%s/checkout-key"
+               repo-name)))
+
+(defn project-checkout-key  [vcs-type repo-name fingerprint]
+  (case vcs-type
+    "bitbucket" (gstring/format
+                  "/api/dangerzone/project/%s/%s/checkout-key/%s"
+                  vcs-type repo-name fingerprint)
+    "github" (gstring/format
+               "/api/v1/project/%s/checkout-key/%s"
+               repo-name fingerprint)))

 (defn project-plan [vcs-type org repo]
   (case vcs-type


Switch to advanced ClojureScript optimizations.

--- metlog-vault/project.clj
+++ metlog-vault/project.clj
@@ -27,8 +27,7 @@

   :cljsbuild {:builds [{:source-paths ["src-client"]
                         :compiler {:output-to "resources/public/metlog.js"
-                                   :optimizations :whitespace
-                                   :pretty-print true}}]}
+                                   :optimizations :advanced}}]}

   :jar-name "metlog-vault.jar"
   :uberjar-name "metlog-vault-standalone.jar")


CMR-484: added search granules by legacy cloud_cover min / max method

--- test/cmr/system_int_test/search/granule_search_test.clj
+++ test/cmr/system_int_test/search/granule_search_test.clj
@@ -243,6 +243,59 @@
                 :errors [(msg/invalid-numeric-range-msg num-range)]}
                (search/find-refs :granule {"cloud_cover" num-range})))))))

+;; covers all of the conditions in search-by-cloud-cover test
+(deftest search-by-legacy-cloud-cover
+  (let [coll1 (d/ingest "CMR_PROV1" (dc/collection {}))
+        coll2 (d/ingest "CMR_PROV2" (dc/collection {}))
+        gran1 (d/ingest "CMR_PROV1" (dg/granule coll1 {:cloud-cover 0.8}))
+        gran2 (d/ingest "CMR_PROV1" (dg/granule coll1 {:cloud-cover 30.0}))
+        gran3 (d/ingest "CMR_PROV1" (dg/granule coll1 {:cloud-cover 120}))
+        gran4 (d/ingest "CMR_PROV2" (dg/granule coll2 {:cloud-cover -60.0}))
+        gran5 (d/ingest "CMR_PROV2" (dg/granule coll2 {:cloud-cover 0.0}))
+        gran6 (d/ingest "CMR_PROV2" (dg/granule coll2 {:granule-ur "sampleur3"}))]
+    (index/flush-elastic-index)
+    (testing "search granules with lower bound cloud-cover value"
+      (are [cc-search items] (d/refs-match? items (search/find-refs :granule cc-search))
+           {"cloud_cover[min]" "0.2"} [gran1 gran2 gran3]))
+    (testing "search granules with upper bound cloud-cover value"
+      (are [cc-search items] (d/refs-match? items (search/find-refs :granule cc-search))
+           {"cloud_cover[max]" "0.7"} [gran4 gran5]))
+    (testing "search by cloud-cover range values that would not cover all granules in store"
+      (are [cc-search items] (d/refs-match? items (search/find-refs :granule cc-search))
+           {"cloud_cover[min]" "-70.0",  "cloud_cover[max]" "31.0"} [gran1 gran2 gran4 gran5]))
+    (testing "search by cloud-cover range values that would not cover all granules in store"
+      (are [cc-search items] (d/refs-match? items (search/find-refs :granule cc-search))
+           {"cloud_cover[min]" "-70.0",  "cloud_cover[max]" "120.0"} [gran1 gran2 gran3 gran4 gran5]))
+    (testing "search by cloud-cover with min value greater than max value"
+      (let [min-value 30.0
+            max-value 0.0]
+        (is (= {:status 422
+                :errors [(vmsg/min-value-greater-than-max min-value max-value)]}
+               (search/find-refs :granule {"cloud_cover[min]" min-value  "cloud_cover[max]" max-value})))))
+    (testing "search by cloud-cover with non numeric strs 'c9c' '' "
+      (let [num-range-p1 "c9c"
+            num-range-p2 ""]
+        (is (= {:status 422
+                :errors [(msg/invalid-numeric-range-msg (format "%s,%s" num-range-p1 num-range-p2))]}
+               (search/find-refs :granule {"cloud_cover[min]" num-range-p1  "cloud_cover[max]" num-range-p2})))))
+    (testing "search by cloud-cover with non numeric strs  '' '99c'"
+      (let [num-range-p1 ""
+            num-range-p2 "99c"]
+        (is (= {:status 422
+                :errors [(msg/invalid-numeric-range-msg (format "%s,%s" num-range-p1 num-range-p2))]}
+               (search/find-refs :granule {"cloud_cover[min]" num-range-p1  "cloud_cover[max]" num-range-p2})))))
+    (testing "search by cloud-cover with empty strs '' ''"
+      (let [num-range-p1 ""
+            num-range-p2 ""]
+        (is (= {:status 422
+                :errors [(msg/invalid-numeric-range-msg (format "%s,%s" num-range-p1 num-range-p2))]}
+               (search/find-refs :granule {"cloud_cover[min]" num-range-p1  "cloud_cover[max]" num-range-p2})))))
+    (testing "search by cloud-cover with invalid range values"
+      (let [num-range-p1 "30"
+            num-range-p2 "c9c"]
+        (is (= {:status 422
+                :errors [(msg/invalid-numeric-range-msg (format "%s,%s" num-range-p1 num-range-p2))]}
+               (search/find-refs :granule {"cloud_cover[min]" num-range-p1  "cloud_cover[max]" num-range-p2})))))))

 ;; Find granules by echo_granule_id, echo_collection_id and concept_id params
 (deftest echo-granule-id-search-test


more progress on warehouse

more progress on warehouse

--- src/Miscellaneous/warehouse.clj
+++ src/Miscellaneous/warehouse.clj
@@ -16,7 +16,7 @@

 (mg/set-db! (mg/get-db "test"))

-(declare extract-text)
+(declare extract-text extract-tieba)

 (defn integrate-text
   [& {:as source}]
@@ -30,19 +30,36 @@
 (defn extract-text
   [source-key source-address]
   (cond (= source-key :tianya) (println "This is tianya")
-        (= source-key :tieba) (mc/find-maps source-address)
+        (= source-key :tieba) (extract-tieba source-address)
         (= source-key :weibo) (println "This is weibo")
         (= source-key :douban) (println "This is douban")
         (= source-key :youku) (println "This is youku")))

-(defn extract-tianya
+(defn extract-tieba
   [source-address]
-  (let [m (mc/find-maps source-address)])
-  (select-keys)
-  )
+  (let [m (mc/find-maps source-address)
+        f #(map :text (:minireps %))
+        g (fn [i] (update-in i [:minireps] (partial apply str)))
+        h (fn [i] [(select-keys i [:_id :minireps])
+                   (select-keys i [:_id :text])])
+        j (fn [i] [(assoc {} :mid (:_id (first i)) :text (:minireps (first i)) :level (Integer. 1))
+                   (assoc {} :mid (:_id (second i)) :text (:text (second i)) :level (Integer. 0))])]
+    (->> m
+         (map #(select-keys % [:_id :text :minireps]))
+         (map #(assoc % :minireps (f %)))
+         (map g)
+         (map h)
+         (map j)
+         flatten
+         (remove #(= "" (:text %)))
+         (map #(assoc % :source "tianya"))
+         )))

+(mc/insert-batch "xuetest" (extract-tieba "star_baidutieba_contents"))

+(extract-tieba "star_baidutieba_contents")

+((comp :text :minireps) {:minireps {:text '3}})

 (mc/find-maps "star_baidutieba_contents")
@@ -52,6 +69,6 @@

+(flatten [[{:a 2} {:b 3}] [{:c 4} {:d 5}]])

-


Remove an unneeded comment.

Remove an unneeded comment.

--- test/picard/test/http/server.clj
+++ test/picard/test/http/server.clj
@@ -1359,5 +1359,4 @@

     (is (closed-socket?))))

-;; (defcoretest connections-marked-as-upgrade-are-not-kept-alive)
 ;; (defcoretest upgraded-connections-do-not-timeout)


refactor the test cases

refactor the test cases

--- test/dalap/test/html.clj
+++ test/dalap/test/html.clj
@@ -11,38 +11,35 @@

 (def basic-sample-data-str "123abcx 1.2 foo:bar")

+(defn- assert-html
+  ([in out] (assert-html in out nil))
+  ([in out msg] (is (= (to-html in) out) msg)))
+
 (deftest test-basic-types
-  (is (= (to-html basic-sample-data)
-         basic-sample-data-str))
-  (is (= (to-html ["" :ti&tle "fo\"o"])
-         ":ti&tlefo"o")))
+  (assert-html basic-sample-data basic-sample-data-str)
+
+  (assert-html ["" :ti&tle "fo\"o"] ":ti&tlefo"o"
+               "the keyword above should be HTML escaped"))

 (deftest test-vector-tag-dsl
-  (is (= (to-html [:hr#id])
-         "
"))
-
-  (is (= (to-html [:hr.classA])
-         "
"))
-
-  (is (= (to-html [:hr#id.cls])
-         "
"))
-
-  (is (= (to-html [:div [:div basic-sample-data]])
-         (str "

" basic-sample-data-str "

")))
-
-  (is (= (to-html [:hr.clear {:asdf 1234}])
-         "
"))
-  (is (= (to-html [:input {:type "text"}]) "<input type=\"text\" />"))
-  (is (= (to-html [:div [:div [:div [:div]]]])
-         "

"))
-  (is (= (to-html [:div#a [:div#b [:div#c [:div#d]]]])
-         (str
-          "

"
-          "

")))
-  (is (= (to-html [:form#contact {:action "GET"}
-                   [:input {:type "text"
-                            :name "first-name"
-                            :value "Tavis & Roman"}]])
-         (str
-          "<form action=\"GET\" id=\"contact\"><input value=\"Tavis &"
-          " Roman\" type=\"text\" name=\"first-name\" /></form>"))))
+  (assert-html [:hr#id] "
")
+  (assert-html [:hr.classA] "
")
+  (assert-html [:hr#id.cls] "
")
+  (assert-html [:div [:div basic-sample-data]]
+               (str "

" basic-sample-data-str "

"))
+  (assert-html [:hr.clear {:asdf 1234}]
+               "
")
+  (assert-html [:input {:type "text"}] "<input type=\"text\" />")
+  (assert-html [:div [:div [:div [:div]]]]
+               "

")
+  (assert-html [:div#a [:div#b [:div#c [:div#d]]]]
+               (str
+                "

"
+                "

"))
+  (assert-html [:form#contact {:action "GET"}
+                [:input {:type "text"
+                         :name "first-name"
+                         :value "Tavis & Roman"}]]
+               (str
+                "<form action=\"GET\" id=\"contact\"><input value=\"Tavis &"
+                " Roman\" type=\"text\" name=\"first-name\" /></form>")))


Adding aRuleForDivisibilityBy7.clj kata. Still need to finish it.

--- hackerrank/functional_programming/clojure/codewars/aRuleForDivisibilityBy7.clj
+++ hackerrank/functional_programming/clojure/codewars/aRuleForDivisibilityBy7.clj
@@ -26,22 +26,29 @@

 (defn is-divisible-by-7 [n]
   "Returns true if n % 7 == 0"
-  (= (= (mod n 7) 0) 0))
+  (= (mod n 7) 0))
+
+(defn is-within-limit [n]
+  "Checks whether a number is between 0 and 99"
+  (and (<= n 99) (>= n 0)))

 (defn seven [m]
-  (loop [steps 1]
-    (let [x (quot m 10)
-          y (* (mod m 10) 2)
-          res (- x y)]
-      (if (is-divisible-by-7 res)
-          (list res steps)))
-      ;; (if-not (and (>= x 0) (< x 99))
-      ;; )
-    (recur (inc steps))))
+  (loop [steps 0 value m]
+    ;; (println "steps: " steps " value: " value)
+    (if (is-within-limit value)
+      (if (is-divisible-by-7 value)
+        (list value steps)
+        (list value steps))
+        ;; (str "Limited reached: " value " at step: " steps))
+      (let [x (quot value 10) y (* (mod value 10) 2) res (- x y)]
+        ;; (println "x: " x " y: " y " res: " res)
+        (recur (inc steps) res)))))

 ;; testing
 (let [v 371 x (seven v) y [35 1]] (println v ": " x "=> " y "? " (= x y)))
 (let [v 1603 x (seven v) y [7 2]] (println  v ": " x "=> " y "? " (= x y)))
+(let [v 477557101 x (seven v) y [28 7]] (println  v ": " x "=> " y "? " (= x y)))
+(let [v 2937633 x (seven v) y [18 5]] (println  v ": " x "=> " y "? " (= x y)))


Some tests added.

Some tests added.

--- KataTrigrams/test/kata_trigrams/test/core.clj
+++ KataTrigrams/test/kata_trigrams/test/core.clj
@@ -1,6 +1,54 @@
 (ns kata-trigrams.test.core
-  (:use [kata-trigrams.core])
+  (:use [kata-trigrams.core :as kt])
   (:use [clojure.test]))

-(deftest test-test
-  (is (= 42 (+ 40 2) "40+2=42")))
+;; test kt/str->trigrams
+(deftest test-str->trigrams
+
+  ;; empty string
+  (is (= ()
+         (kt/str->trigrams "")))
+
+  ;; one word (no trigram)
+  (is (= ()
+         (kt/str->trigrams "one")))
+
+  ;; two word (no trigram)
+  (is (= ()
+         (kt/str->trigrams "one two")))
+
+  ;; three words -> one trigram
+  (is (= '(("one" "two" "three"))
+         (kt/str->trigrams "one two three")))
+
+  ;; four words -> two trigrams
+  (is (= '(("one" "two" "three") ("two" "three" "four"))
+         (kt/str->trigrams "one two three four")))
+
+  ;; punctuation
+  (is (= '(("a" "b" "c."))
+         (kt/str->trigrams "a b c.")))
+
+  ;; hyphens in words
+  (is (= '(("a" "b-b" "c"))
+         (kt/str->trigrams "a b-b c"))))
+
+
+;; test kt/add-trigram
+(deftest test-add-trigram
+
+  ;; not a trigram
+  (is (= {}
+         (kt/add-trigram {} '("one"))))
+  (is (= {}
+         (kt/add-trigram {} '("one" "two"))))
+
+  ;; trigram with first two words not in the map
+  (is (= {"a b" #{"c"}}
+         (kt/add-trigram {} '("a" "b" "c"))))
+  (is (= {"a b" #{"c"} "a c" #{"b"}}
+         (kt/add-trigram {"a c" #{"b"}} '("a" "b" "c"))))
+
+  ;; trigram with first two words already in the map
+  (is (= {"a b" #{"c" "d"}}
+         (kt/add-trigram {"a b" #{"c"}} '("a" "b" "d")))))


Fix broken API doc cross-reference.

Fix broken API doc cross-reference.

--- src/afterglow/effects/color.clj
+++ src/afterglow/effects/color.clj
@@ -154,7 +154,7 @@
 ;;; Effects which transform other color effects

 (defn build-saturation-transformation
-  "Creates a color transformation for use with [[transform-color]]
+  "Creates a color transformation for use with [[transform-colors]]
   which changes the saturation based on a variable parameter. If no
   parameter is supplied, the default is to use an oscillated parameter
   based on [[sawtooth-beat]] with `:down?` set to `true` so the color


s/map-subset-by-{pred,cat}/map-subset-by-path/

--- src/babel/francais/grammar.cljc
+++ src/babel/francais/grammar.cljc
@@ -7,8 +7,8 @@
    [babel.francais.lexicon :refer [lexicon]]
    [babel.francais.morphology :as morph :refer [fo]]
    [babel.index :refer (build-lex-sch-index
-                        create-index map-subset-by-cat
-                        map-subset-by-pred spec-to-phrases)]
+                        create-index map-subset-by-path
+                        spec-to-phrases)]
    [babel.parse :as parse]
    [babel.ug :refer [comp-modifies-head comp-specs-head head-principle
                      root-is-comp root-is-head-root root-is-head
@@ -489,20 +489,10 @@
      :enrich enrich

      :pred2lex ;; map:<pred => subset of lexicon with that pred>
-     (map-subset-by-pred
-      (filter #(not (nil? %))
-              (vec (set (mapcat (fn [entry]
-                                  (get-in entry [:synsem :sem :pred]))
-                                (vals lexicon)))))
-      (flatten (vals lexicon)))
+     (map-subset-by-path lexicon-for-generation [:synsem :sem :pred])

      :cat2lex ;; map:<cat => subset of lexicon with that cat>
-     (map-subset-by-cat
-      (filter #(not (nil? %))
-              (vec (set (mapcat (fn [entry]
-                                  (get-in entry [:synsem :cat]))
-                                (vals lexicon)))))
-      (flatten (vals lexicon)))
+     (map-subset-by-path lexicon-for-generation [:synsem :cat])

      :grammar grammar
      ;; Will throw exception if more than 1 rule has the same :rule value:
@@ -561,20 +551,10 @@
                                  (morph-walk-tree tree))))

      :pred2lex ;; map:<pred => subset of lexicon with that pred>
-     (map-subset-by-pred
-      (filter #(not (nil? %))
-              (vec (set (mapcat (fn [entry]
-                                  (get-in entry [:synsem :sem :pred]))
-                                (vals lexicon)))))
-      (flatten (vals lexicon)))
+     (map-subset-by-path lexicon [:synsem :sem :pred])

      :cat2lex ;; map:<cat => subset of lexicon with that cat>
-     (map-subset-by-cat
-      (filter #(not (nil? %))
-              (vec (set (mapcat (fn [entry]
-                                  (get-in entry [:synsem :cat]))
-                                (vals lexicon)))))
-      (flatten (vals lexicon)))
+     (map-subset-by-path lexicon [:synsem :cat])

      :morph-ps fo-ps
      :language "fr"


fix typo

fix typo

--- src/babel/italiano/lexicon.cljc
+++ src/babel/italiano/lexicon.cljc
@@ -1141,7 +1141,7 @@
              common-noun
              countable-noun
              masculine-noun
-             {:synsem {:sem {:animal true
+             {:synsem {:sem {:animate true
                              :human false
                              :pred :gatto
                              :pet true}}})
@@ -2130,4 +2130,3 @@
                     (filter #(or (not (= :verb (get-in % [:synsem :cat])))
                                  (not (= :none (get-in % [:synsem :infl] :none))))
                             vals)))))
-


make initialization faster

make initialization faster

--- src/babel/italiano/workbook.clj
+++ src/babel/italiano/workbook.clj
@@ -173,6 +173,9 @@
        (if search-query
          (workbookq search-query))]])))

+;; not sure why this is necessary: investigate why english/workbook.clj doesn't need it.
+(def foo (count (morph/analyze-regular "parlo" lexicon)))
+
 (def routes
   (compojure/routes

@@ -192,9 +195,3 @@
          :headers {"Content-Type" "text/html;charset=utf-8"}})
   ))

-;; not sure why this is necessary: investigate why english/workbook.clj doesn't need it.
-(def foo (count (take 1 (parse "io dormo"))))
-(def foo2 (count (take 1 (generate {:synsem {:sem {:pred :sleep
-                                                   :tense :present
-                                                   :subj {:pred :I}}}}))))
-


first working fix-time-series function; time to clean up and document.

--- src/clj/forma/trends/filter.clj
+++ src/clj/forma/trends/filter.clj
@@ -89,6 +89,7 @@
                          (partition 2 1 goodpos-seq))
                     (nth value-coll (last goodpos-seq))))))))

+
 (defn mask
   "create a new vector where values from `coll1` are only passed through
   if they satisfy the predicate `pred` for `coll2`.  All other values are
@@ -97,10 +98,16 @@
   {:pre [(= (count coll1) (count coll2))]}
   (map #(when-not (pred %2) %1) coll2 coll1))

-(defn fix-time-series
-  [pred qual-coll val-coll]
-  (map-indexed vector (mask pred qual-coll val-coll)))
+;; (defn fix-time-series
+;;   [pred qual-coll val-coll]
+;;   (vec
+;;    (map-indexed vector (mask pred qual-coll val-coll))))

+
+(defn replace-index-set
+  [idx-set new-val coll]
+  (for [[m n] (map-indexed vector coll)]
+    (if (idx-set m) new-val n)))

 (defn bad-ends
   "collect a set of the indices of bad ends. if the bad value is, say, 2, then the
@@ -114,18 +121,32 @@
                  [m-coll r-coll])))))

 (defn act-on-good
-  [func bad-set coll]
+  [func coll]
   (func
-   (filter #(not (contains? bad-set %))
-           coll)))
+   (filter (complement nil?) coll)))

 (defn neutralize-ends
   [bad-set reli-coll val-coll]
-  (let [avg (act-on-good average bad-set val-coll)
-        ends (bad-ends bad-set reli-coll)]
-    ends))
+  (let [avg (act-on-good average
+                         (mask bad-set reli-coll val-coll))]
+    (replace-index-set
+     (bad-ends bad-set reli-coll)
+     avg
+     val-coll)))

-;; (map #(if (#{8049} %) 54 %) (mask #{2} reli-test ndvi-test))
+(defn fix-time-series
+  ""
+  [bad-val-set quality-coll value-coll]
+  (if (empty? (filter bad-val-set quality-coll)) nil
+      (let [bad-end-set (bad-ends bad-val-set quality-coll)
+            new-qual (replace-index-set bad-end-set 1 quality-coll)
+            new-vals (neutralize-ends bad-val-set quality-coll value-coll)
+            goodpos-seq (positions (complement bad-val-set) new-qual)]
+        (vec (flatten
+              (vector (map (partial stretch-testing new-vals)
+                           (partition 2 1 goodpos-seq))
+                      (nth new-vals (last goodpos-seq))))))))
+

 (def reli-test [2 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 2])


update revoked badge visibility

update revoked badge visibility

--- src/clj/salava/badge/verify.clj
+++ src/clj/salava/badge/verify.clj
@@ -75,7 +75,6 @@
                               :revoked? true))
           500 (assoc result :assertion-status 500
                             :asr asr
-                            :badge-status "Broken assertion url, badge can't be verified"
                             :message (:message asr-response))
           200 (let [asr-data (:body asr-response)
                     badge-data (fetch-json-data (:badge asr-data))
@@ -104,7 +103,6 @@
         (if (= 500 (:status jws-response))
           (assoc result :assertion-status 500
                         :asr asr
-                        :badge-status "Broken jws signature, badge can't be verified"
                         :message (:message jws-response))
           (let [badge-id (or (:id (json/read-str (:assertion_json jws-response) :key-fn keyword))(:uid (json/read-str (:assertion_json jws-response) :key-fn keyword)))
                 revocation-list-url (:revocation_list_url (first (get-in jws-response [:badge :issuer])))
@@ -115,7 +113,8 @@
                 issuedOn {:issuedOn (process-time (:issuedOn assertion))}
                 expires (if-let [exp (process-time (:expires assertion))]{:expires exp} nil)]

-            (if (not (empty? revoked))(update-revoked! {:revoked 1 :id (:id badge)} (get-db ctx)))
+            (if (not (empty? revoked))(do (update-revoked! {:revoked 1 :id (:id badge)} (get-db ctx))
+                                          (update-visibility! {:visibility "private" :id (:id badge)} (get-db ctx))))

             (assoc result :assertion-status 200
                           :assertion (merge assertion issuedOn expires)


clean code

clean code

--- src/clj/salava/oauth/routes.clj
+++ src/clj/salava/oauth/routes.clj
@@ -43,7 +43,6 @@
                                  {error :- s/Str nil}]
                   :current-user current-user
                   (let [{:keys [status user-id message role private]} (f/facebook-login ctx code (:id current-user) error)
-                        gdpr-disabled? (first (mapcat #(get-in ctx [:config % :disable-gdpr] []) (get-plugins ctx)))
                         _ (if (= true (get-in req [:session :seen-terms])) (d/insert-user-terms ctx user-id "accepted"))
                         accepted-terms? (u/get-accepted-terms-by-id ctx user-id)]
                     (if (= status "success")