CMR-4239 - Removed obsolete code.

CMR-4239 - Removed obsolete code.

--- system-int-test/src/cmr/system_int_test/utils/variable_util.clj
+++ system-int-test/src/cmr/system_int_test/utils/variable_util.clj
@@ -86,87 +86,6 @@
   [attrs]
   (ingest-variable (make-variable-concept attrs)))

-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(def sample-variable
-  {:Name "A-name"
-   :LongName "A long UMM-Var name"
-   :Units "m"
-   :DataType "float32"
-   :DimensionsName "H2OFunc"
-   :Dimensions "11"
-   :ValidRange {}
-   :Scale "1.0"
-   :Offset "0.0"
-   :FillValue "-9999.0"
-   :VariableType "SCIENCE_VARIABLE"
-   :ScienceKeywords [{:Category "sk-A"
-                       :Topic "sk-B"
-                       :Term "sk-C"}]})
-
-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(defn make-variable
-  "Makes a valid variable based on the given input"
-  ([]
-   (make-variable nil))
-  ([attrs]
-   (merge sample-variable attrs))
-  ([index attrs]
-   (merge
-    sample-variable
-    {:Name (str "Name" index)
-     :LongName (str "Long UMM-Var name " index)}
-    attrs)))
-
-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(defn create-variable
-  "Creates a variable."
-  ([token variable]
-   (create-variable token variable nil))
-  ([token variable options]
-   (let [options (merge {:raw? true
-                         :token token
-                         :http-options {:content-type "application/vnd.nasa.cmr.umm+json"}}
-                        options)]
-     (ingest-util/parse-map-response
-      (transmit-variable/create-variable (s/context) variable options)))))
-
-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(defn create-variable-with-attrs
-  "Helper function to create a variable with the given variable attributes"
-  [token attrs]
-  (create-variable token (make-variable attrs)))
-
-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(defn update-variable
-  "Updates a variable."
-  ([token variable]
-   (update-variable token (:variable-name variable) variable nil))
-  ([token variable-name variable]
-   (update-variable token variable-name variable nil))
-  ([token variable-name variable options]
-   (let [options (merge {:raw? true
-                         :token token
-                         :http-options {:content-type "application/vnd.nasa.cmr.umm+json"}}
-                        options)]
-     (ingest-util/parse-map-response
-      (transmit-variable/update-variable (s/context) variable-name variable options)))))
-
-;; XXX This can be removed once variable associations have been updated to use the new
-;; cmr.system-int-test.data2.umm-spec-variable namespace.
-(defn delete-variable
-  "Deletes a variable"
-  ([token variable-name]
-   (delete-variable token variable-name nil))
-  ([token variable-name options]
-   (let [options (merge {:raw? true :token token} options)]
-     (ingest-util/parse-map-response
-      (transmit-variable/delete-variable (s/context) variable-name options)))))
-
 (defn- associate-variable
   "Associate a variable with collections by the JSON condition.
   Valid association types are :query and :concept-ids."
@@ -213,30 +132,6 @@
    (dissociate-by-concept-ids token variable-name coll-concept-ids nil))
   ([token variable-name coll-concept-ids options]
    (dissociate-variable :concept-ids token variable-name coll-concept-ids options)))
-
-(defn save-variable
-  "A helper function for creating or updating variables for search tests.
-   If the variable does not have a :concept-id it saves it. If the variable has a :concept-id,
-   it updates the variable. Returns the saved variable along with :concept-id,
-   :revision-id, :errors, and :status"
-  ([token variable]
-   (let [variable-to-save (select-keys variable [:variable-name :description :revision-date])
-         response (if-let [concept-id (:concept-id variable)]
-                    (update-variable token (:variable-name variable) variable-to-save)
-                    (create-variable token variable-to-save))
-         variable (into variable
-                        (select-keys response [:status :errors :concept-id :revision-id]))]
-     (if (= (:revision-id variable) 1)
-       ;; Get the originator id for the variable
-       (assoc variable :originator-id (tokens/get-user-id (s/context) token))
-       variable)))
-  ([token variable associated-collections]
-   (let [saved-variable (save-variable token variable)
-         ;; Associate the variable with the collections using a query by concept id
-         condition {:or (map #(hash-map :concept_id (:concept-id %)) associated-collections)}
-         response (associate-by-query token (:variable-name saved-variable) condition)]
-     (assert (= 200 (:status response)) (pr-str condition))
-     (assoc saved-variable :revision-id (:revision-id response)))))

 (defn expected-concept
   "Create an expected concept given a service, its concept-id, a revision-id,


Moved things around a bit

Moved things around a bit

--- test/cluster_tests.clj
+++ test/cluster_tests.clj
@@ -2,6 +2,10 @@
   (:require [clojure.test :refer :all]
             [polismath.named-matrix :refer :all]
             [polismath.clusters :refer :all]))
+
+(defn size-correct [clusters n]
+  (is (= n (count clusters))))
+

 (def real-nmat (named-matrix
   ["p1" "p2" "p3" "p4" "p5" "p6"]
@@ -12,10 +16,6 @@
    [ 1  0  1  0]
    [-1 -1  0  1]
    [-1  1  0  1]]))
-
-
-(defn size-correct [clusters n]
-  (is (= n (count clusters))))

 ; Initialization test


Test showing only articles with images.

Test showing only articles with images.

--- src/wlmmap/handler.clj
+++ src/wlmmap/handler.clj
@@ -40,7 +40,7 @@
 (defn- generate-markers
   "return a string with the javascript code to generate addressPoints."
   [params]
-  (let [mons (take 2000 (wcar* (car/smembers "frfr")))
+  (let [mons (take 2000 (wcar* (car/smembers "frfrim")))
         all (remove #(nil? %)
                     (for [m mons]
                       (let [res (read-string (wcar* (car/get m)))


Implement :option :option-reset guide commands.

--- src/zprint/zprint.cljc
+++ src/zprint/zprint.cljc
@@ -1038,48 +1038,41 @@
                 flow-indent (+ indent ind)]
             (if (and (zstring lloc)
                      (keyword-fn? options (zstring lloc))
-		     (not (= caller :map)))
-		     ; We could also check for (= caller :pair-fn) here,
-		     ; or at least check to see that it isn't a map.
-
-	       (if (zvector? rloc)
-              ; This is an embedded :let or :when-let or something.
-	      ; We check to see if a keyword is found in the :fn-map
-	      ; (without the :, of course) and if it is and there
-	      ; is a vector after it, we assume that it must be one of these.
-              (let [[hang-or-flow style-vec] (fzprint-hang-unless-fail
-                                               loptions
-                                               hanging-indent
-                                               flow-indent
-                                               fzprint-binding-vec
-                                               rloc)
-                    arg-1 (if (= hang-or-flow :hang)
-                            (concat-no-nil arg-1
-                                           [[(blanks hanging-spaces) :none
-                                             :whitespace 2]])
-                            arg-1)]
-                [hang-or-flow (concat-no-nil arg-1 style-vec)])
-              (let [[hang-or-flow style-vec] (fzprint-hang-unless-fail
-                                               loptions
-                                               hanging-indent
-                                               flow-indent
-                                               fzprint*
-                                               rloc)
-                    arg-1 (if (= hang-or-flow :hang)
-                            (concat-no-nil arg-1
-                                           [[(blanks hanging-spaces) :none
-                                             :whitespace 2]])
-                            arg-1)]
-                [hang-or-flow (concat-no-nil arg-1 style-vec)])
-
-
-		)
-
-
+                     (not (= caller :map)))
+              ; We could also check for (= caller :pair-fn) here,
+              ; or at least check to see that it isn't a map.
+              (if (zvector? rloc)
+                ; This is an embedded :let or :when-let or something.
+                ; We check to see if a keyword is found in the :fn-map
+                ; (without the :, of course) and if it is and there
+                ; is a vector after it, we assume that it must be one of these.
+                (let [[hang-or-flow style-vec] (fzprint-hang-unless-fail
+                                                 loptions
+                                                 hanging-indent
+                                                 flow-indent
+                                                 fzprint-binding-vec
+                                                 rloc)
+                      arg-1 (if (= hang-or-flow :hang)
+                              (concat-no-nil arg-1
+                                             [[(blanks hanging-spaces) :none
+                                               :whitespace 2]])
+                              arg-1)]
+                  [hang-or-flow (concat-no-nil arg-1 style-vec)])
+                (let [[hang-or-flow style-vec] (fzprint-hang-unless-fail
+                                                 loptions
+                                                 hanging-indent
+                                                 flow-indent
+                                                 fzprint*
+                                                 rloc)
+                      arg-1 (if (= hang-or-flow :hang)
+                              (concat-no-nil arg-1
+                                             [[(blanks hanging-spaces) :none
+                                               :whitespace 2]])
+                              arg-1)]
+                  [hang-or-flow (concat-no-nil arg-1 style-vec)]))
               ; Make the above if a cond, and call fzprint-hang-one?  Or
-	      ; maybe fzprint* if we are calling fzprint-hang-unless-fail,
-	      ; which I think we are.
-
+              ; maybe fzprint* if we are calling fzprint-hang-unless-fail,
+              ; which I think we are.
               ; This is a normal two element pair thing
               (let [; Perhaps someday we could figure out if we are already
                     ; completely in flow to this point, and be smarter about
@@ -5176,7 +5169,7 @@
              [previous-newline? previous-guided-newline? unguided-newline-out?
               previous-comment? :as previous-data]
                nil
-             ; transient here slows things down, interestingly enough
+             options options
              out []]
         ; We can't just check for cur-seq here, or any pairs we might be
         ; accumulating will be lost, since :pair-end has to come beyond
@@ -5244,6 +5237,7 @@
                            (assoc param-map :pair-seq pair-seq)
                            mark-map
                            previous-data
+                           options
                            out))
                 (= (first guide-seq) :pair-end)
                   ; output accumulated pairs
@@ -5271,6 +5265,7 @@
                                  (dissoc param-map :pair-seq)
                                  mark-map
                                  previous-data
+                                 options
                                  out)
                           ; we got output from guide-pairs
                           (let [[new-param-map new-previous-data new-out]
@@ -5297,6 +5292,7 @@
                                    (dissoc new-param-map :pair-seq)
                                    mark-map
                                    new-previous-data
+                                   options
                                    new-out))))
                     (throw
                       (#?(:clj Exception.
@@ -5334,6 +5330,7 @@
                              [previous-newline? previous-guided-newline?
                               ;unguided-newline-out?
                               nil previous-comment?]
+                             options
                              out))
                 (= (first guide-seq) :mark)
                   ; put the cur-ind into the mark map with key the next
@@ -5358,6 +5355,7 @@
                                                          (or (:spaces param-map)
                                                              1)))
                            previous-data
+                           options
                            out))
                 (= (first guide-seq) :spaces)
                   ; save the spaces for when we actually do output
@@ -5375,6 +5373,7 @@
                              (assoc param-map :spaces (first (next guide-seq)))
                              mark-map
                              previous-data
+                             options
                              out))
                 (= (first guide-seq) :indent)
                   ; save a new indent value in param-map
@@ -5391,6 +5390,7 @@
                              (assoc param-map :indent (first (next guide-seq)))
                              mark-map
                              previous-data
+                             options
                              out))
                 (= (first guide-seq) :indent-reset)
                   ; put the indent back where it was originally
@@ -5406,6 +5406,46 @@
                              (assoc param-map :indent local-indent)
                              mark-map
                              previous-data
+                             options
+                             out))
+                (= (first guide-seq) :options)
+                  ; Start using an updated options map
+                  (let [[new-options _ error-vec]
+                          (zprint.config/config-and-validate
+                                   "guide: command :option"
+                                   nil
+                                   options
+                                   (first (next guide-seq)))
+                        _ (dbg-s options
+                                 :guide
+                                 "fzprint-guide: === :options"
+                                 (first (next guide-seq)))]
+                    (recur cur-zloc
+                           cur-index
+                           ; skip an extra to account for the options map
+                           (nnext guide-seq)
+                           element-index
+                           (inc index)
+			   param-map
+                           mark-map
+                           previous-data
+                           new-options
+                           out))
+                (= (first guide-seq) :options-reset)
+                  ; put the options map back to where it was when we started
+                  (do (dbg-s options
+                             :guide
+                             (color-str "fzprint-guide: === :options-reset"
+                                        :bright-red))
+                      (recur cur-zloc
+                             cur-index
+                             (next guide-seq)
+                             element-index
+                             (inc index)
+                             (assoc param-map :indent local-indent)
+                             mark-map
+                             previous-data
+                             (:initial-options param-map)
                              out))
                 (= (first guide-seq) :align)
                   ; Set up for an alignment on the next :element
@@ -5426,6 +5466,7 @@
                            (assoc param-map :align-key align-key)
                            mark-map
                            previous-data
+                           options
                            out))
                 (:guided-newline-count param-map)
                   ; we are currently counting newlines, see if we have more
@@ -5453,6 +5494,7 @@
                                      :align-key)
                              mark-map
                              previous-data
+                             options
                              out))
                     ; we are counting guided-newlines, and we have found a guide
                     ; that is not a :newline, so we need to determine the number
@@ -5524,6 +5566,7 @@
                              new-param-map
                              mark-map
                              new-previous-data
+                             options
                              new-out)))
                 (= (first guide-seq) :newline)
                   ; start counting guided newlines
@@ -5547,6 +5590,7 @@
                                    :align-key)
                            mark-map
                            previous-data
+                           options
                            out))
                 ; :pair-begin has to come after the newline handling
                 (= (first guide-seq) :pair-begin)
@@ -5564,8 +5608,9 @@
                              (assoc param-map :pair-seq [])
                              mark-map
                              previous-data
+                             options
                              out))
-		; :pair-* also has to come after newline handling
+                ; :pair-* also has to come after newline handling
                 (= (first guide-seq) :pair-*)
                   ; Consider everything else pairs, and output them.  If
                   ; we are already accumulating pairs, that's ok (though
@@ -5605,6 +5650,7 @@
                              (dissoc param-map :pair-seq)
                              mark-map
                              previous-data
+                             options
                              out)
                       ; we have pairs to process
                       (let [[new-param-map new-previous-data new-out]
@@ -5631,6 +5677,7 @@
                                (dissoc new-param-map :pair-seq)
                                mark-map
                                new-previous-data
+                               options
                                new-out))))
                 ;
                 ;  Start looking at cur-seq
@@ -5657,6 +5704,7 @@
                              param-map
                              mark-map
                              previous-data
+                             options
                              out))
                 (or comment? comment-inline? next-newline?)
                   ; Do unguided output, moving cur-zloc without changing
@@ -5694,6 +5742,7 @@
                            new-param-map
                            mark-map
                            new-previous-data
+                           options
                            new-out))
                 (or (= (first guide-seq) :element)
                     (= (first guide-seq) :element-multi)
@@ -5745,6 +5794,7 @@
                            (dissoc new-param-map :spaces :align-key)
                            mark-map
                            new-previous-data
+                           options
                            new-out))
                 ;
                 ; Something we didn't expect is going on here


Added util predicates for posts.

Added util predicates for posts.

--- src/dragon/util.clj
+++ src/dragon/util.clj
@@ -3,7 +3,8 @@
             [clojure.java.shell :as shell]
             [clojusc.twig :refer [pprint]]
             [selmer.parser :as selmer]
-            [taoensso.timbre :as log]))
+            [taoensso.timbre :as log])
+  (:refer-clojure :exclude [boolean]))

 (declare merge-val)

@@ -163,3 +164,22 @@
   (if (contains? system-or-component :event)
     system-or-component
     {:event system-or-component}))
+
+(defn boolean
+  "A filter that returns `true` if the post should be published on the front
+  page."
+  [post key]
+  (if (= (string/lower-case (or (key post) "")) "false")
+    false
+    true))
+
+(defn headline?
+  "A filter that returns `true` if the post should be published on the front
+  page."
+  [post]
+  (boolean post :headlines?))
+
+(defn public?
+  "A filter that returns `true` if the post should be published."
+  [post]
+  (boolean post :public?))


new /config endpoint shows configuration

--- src/fxc/handler.clj
+++ src/fxc/handler.clj
@@ -128,6 +128,9 @@
                       [:div {:class "password"}
                        "Your Secret: "
                        [:div {:class "content"} converted]])}))))
+  (GET "/config" []
+       (web/render-static (present/edn->html (config-read settings))))
+
   ;; TODO: detect cryptographical conversion error: returned is the first share

   (route/not-found "Not Found"))


Fixed privacy of make-id function

Fixed privacy of make-id function

--- src/hiccup/form.clj
+++ src/hiccup/form.clj
@@ -18,7 +18,7 @@
   (reduce #(str %1 "[" %2 "]")
           (conj *group* (as-str name))))

-(defn make-id
+(defn- make-id
   "Create a field id from the supplied argument and current field group."
   [name]
   (reduce #(str %1 "-" %2)


[#2071] Add specs for core/derive-category op

--- backend/specs/akvo/lumen/specs/transformation.clj
+++ backend/specs/akvo/lumen/specs/transformation.clj
@@ -40,6 +40,7 @@
 				    "core/filter-column"
 				    "core/generate-geopoints"
 				    "core/merge-datasets"
+                                    "core/derive-category"
 				    "core/remove-sort"
 				    "core/rename-column"
 				    "core/reverse-geocode"
@@ -177,8 +178,6 @@
 (create-ns  'akvo.lumen.specs.transformation.merge-datasets.target)
 (alias 'transformation.merge-datasets.target 'akvo.lumen.specs.transformation.merge-datasets.target)

-
-
 (s/def ::transformation.merge-datasets.source/datasetId ::db.dsv.s/dataset-id)
 (s/def ::transformation.merge-datasets.source/aggregationColumn (s/nilable columnName?))

@@ -207,6 +206,42 @@
    :req-un [::transformation.merge-datasets/args
             ::transformation.engine/op]))

+(create-ns  'akvo.lumen.specs.transformation.derive-category)
+(alias 'transformation.derive-category 'akvo.lumen.specs.transformation.derive-category)
+
+(create-ns  'akvo.lumen.specs.transformation.derive-category.source)
+(alias 'transformation.derive-category.source 'akvo.lumen.specs.transformation.derive-category.source)
+
+(create-ns  'akvo.lumen.specs.transformation.derive-category.target)
+(alias 'transformation.derive-category.target 'akvo.lumen.specs.transformation.derive-category.target)
+
+(create-ns  'akvo.lumen.specs.transformation.derive-category.derivation)
+(alias 'transformation.derive-category.derivation 'akvo.lumen.specs.transformation.derive-category.derivation)
+
+(s/def ::transformation.derive-category.source/columnName ::i.values.s/id)
+(s/def ::transformation.derive-category.source/column (s/keys :req-un [::transformation.derive-category.source/columnName]))
+
+(s/def ::transformation.derive-category/source (s/keys
+                                               :req-un [::transformation.derive-category.source/column]))
+
+(s/def ::transformation.derive-category.target/column (s/keys :req-un [::i.values.s/title]))
+(s/def ::transformation.derive-category/target (s/keys
+                                               :req-un [::transformation.derive-category.target/column]))
+
+(s/def ::transformation.derive-category.derivation/mappings (s/coll-of (s/tuple string? string?) :kind vector?))
+
+(s/def ::transformation.derive-category/derivation
+  (s/keys :req-un [::transformation.derive-category.derivation/mappings]))
+
+(s/def ::transformation.derive-category/args
+  (s/keys :req-un [::transformation.derive-category/source
+                   ::transformation.derive-category/target
+                   ::transformation.derive-category/derivation]))
+
+(defmethod op-spec "core/derive-category"  [_]
+  (s/keys
+   :req-un [::transformation.derive-category/args
+            ::transformation.engine/op]))

 (create-ns 'akvo.lumen.specs.transformation.extract-multiple)
 (alias 'transformation.extract-multiple 'akvo.lumen.specs.transformation.extract-multiple)


More specificity in Eastwood config.

More specificity in Eastwood config.

--- third-party-macros.clj
+++ third-party-macros.clj
@@ -1,6 +1,6 @@
 (disable-warning
  {:linter :suspicious-expression
-  ;; specifically, those detected in function suspicious-macro-invocations
+  :for-macro 'clojure.core/and
   :if-inside-macroexpansion-of #{'defun.core/defun}
-  :within-depth 12
+  :within-depth 20
   :reason "defun creates calls to and with only 1 argument."})


Fixing ummatched delimiter (#279)

Fixing ummatched delimiter (#279)

--- server/test/spacon/generate_data.clj
+++ server/test/spacon/generate_data.clj
@@ -20,7 +20,7 @@
         team (clojure.spec/exercise-fn `spacon.components.team.db/create)
         store (clojure.spec/exercise-fn `spacon.components.store.db/create)
         form (clojure.spec/exercise-fn `spacon.components.form.db/add-form-with-fields)
-        dev (clojure.spec/exercise-fn `spacon.components.device.db/create)
+        dev (clojure.spec/exercise-fn `spacon.components.device.db/create)]
     (+ (count org) (count team) (count store) (count form) (count dev))))

 (defn -main


Simplified spec for integrate-ident

Simplified spec for integrate-ident

--- spec/untangled/client/core_spec.cljs
+++ spec/untangled/client/core_spec.cljs
@@ -109,36 +109,35 @@

 (specification "integrate-ident"
   (let [state {:a {:path [[:table 2]]}
-                  :b     {:path [[:table 2]]}
-                  :d     [:table 6]
-                  :many  {:path [[:table 99] [:table 88] [:table 77]]}
-                  }]
-    (behavior "Can append to an existing vector"
-      (assertions
+                   :b     {:path [[:table 2]]}
+                   :d     [:table 6]
+                   :many  {:path [[:table 99] [:table 88] [:table 77]]}}]
+     (assertions
+       "Can append to an existing vector"
        (-> state
            (uc/integrate-ident [:table 3] :append [:a :path])
            (get-in [:a :path]))
-       => [[:table 2] [:table 3]])
-      (assertions
+       => [[:table 2] [:table 3]]
+
        "(is a no-op if the ident is already there)"
        (-> state
            (uc/integrate-ident [:table 3] :append [:a :path])
            (get-in [:a :path]))
-       => [[:table 2] [:table 3]]))
-    (behavior "Can prepend to an existing vector"
-      (assertions
+       => [[:table 2] [:table 3]]
+
+       "Can prepend to an existing vector"
        (-> state
            (uc/integrate-ident [:table 3] :prepend [:b :path])
            (get-in [:b :path]))
-       => [[:table 3] [:table 2]])
-      (assertions
+       => [[:table 3] [:table 2]]
+
        "(is a no-op if already there)"
        (-> state
            (uc/integrate-ident [:table 3] :prepend [:b :path])
            (get-in [:b :path]))
-       => [[:table 3] [:table 2]]))
-    (behavior "Can create/replace a to-one ident"
-      (assertions
+       => [[:table 3] [:table 2]]
+
+       "Can create/replace a to-one ident"
        (-> state
            (uc/integrate-ident [:table 3] :replace [:d])
            (get-in [:d]))
@@ -146,13 +145,13 @@
        (-> state
            (uc/integrate-ident [:table 3] :replace [:c :path])
            (get-in [:c :path]))
-       => [:table 3]))
-    (behavior "Can replace an existing to-many element in a vector"
-      (assertions
+       => [:table 3]
+
+       "Can replace an existing to-many element in a vector"
        (-> state
            (uc/integrate-ident [:table 3] :replace [:many :path 1])
            (get-in [:many :path]))
-       => [[:table 99] [:table 3] [:table 77]]))))
+       => [[:table 99] [:table 3] [:table 77]])))

 (specification "Untangled Application -- clear-pending-remote-requests!"
   (let [channel (async/chan 1000)


RealTRMatrix submatrix implemented.

RealTRMatrix submatrix implemented.

--- src/clojure/uncomplicate/neanderthal/impl/buffer_block.clj
+++ src/clojure/uncomplicate/neanderthal/impl/buffer_block.clj
@@ -607,7 +607,8 @@
                        ^long ord ^long fuplo ^long fdiag]
   Object
   (hashCode [this]
-    (matrix-fold n col-row* hash* (-> (hash :RealTRMatrix) (hash-combine n)) this));; TODO do with suubmatrix (.submatrix this 0 0 (min 16 n) (min 16 n))
+    (matrix-fold n col-row* hash* (-> (hash :RealTRMatrix) (hash-combine n))
+                 (.submatrix this 0 0 (min 16 n) (min 16 n))))
   (equals [a b]
     (cond
       (nil? b) false
@@ -656,7 +657,7 @@
     (raw this fact))
   (host [this]
     (let-release [res (raw this)]
-      (.copy eng this res)));;TODO implement TR.copy
+      (.copy eng this res)))
   (native [this]
     this)
   MemoryContext
@@ -739,7 +740,12 @@
       (.subvector ^Vector (col* fact buf ld n n j)
                   start (- (.invokePrim col-end* n j) start))))
   (submatrix [a i j k l]
-    (throw (UnsupportedOperationException. "TODO I should return banded matrix or GE, or consider only TR submatrices valid.")))
+    (if (and (= i j) (= k l))
+      (real-tr-matrix fact false
+                      (.slice da buf (.invokePrim index* ld i j)
+                              (.invokePrim index* ld k (dec l)))
+                      k ld ord fuplo fdiag))
+    (throw (UnsupportedOperationException. "Submatrix of a TR matrix has to be triangular.")))
   (transpose [a]
     (real-tr-matrix fact false buf n ld
                     (if (= COLUMN_MAJOR ord) ROW_MAJOR COLUMN_MAJOR)


Organize cortex.util and fix formatting

Organize cortex.util and fix formatting

--- cortex/src/cortex/util.cljc
+++ cortex/src/cortex/util.cljc
@@ -12,52 +12,7 @@
 #?(:clj (do (set! *warn-on-reflection* true)
             (set! *unchecked-math* :warn-on-boxed)))

-
-#?(:clj (defn timestamp [] (System/nanoTime))
-   :cljs (defn timestamp [] (.getTime (js/Date.))))
-
-(def EMPTY-VECTOR (m/new-array [0]))
-
-#?(:clj
-    (do
-      (def ^Random RAND-GENERATOR (Random.))
-
-      (defn rand-normal ^double []
-        (.nextDouble RAND-GENERATOR))
-
-      (defn rand-gaussian ^double []
-        (.nextGaussian RAND-GENERATOR)))
-    :cljs
-    (do
-      (defn rand-gaussian* [mu sigma]
-        ; This function implements the Kinderman-Monahan ratio method:
-        ;  A.J. Kinderman & J.F. Monahan
-        ;  Computer Generation of Random Variables Using the Ratio of Uniform Deviates
-        ;  ACM Transactions on Mathematical Software 3(3) 257-260, 1977
-        (let [u1  (rand)
-              u2* (rand)
-              u2 (- 1. u2*)
-              s (* 4 (/ (Math/exp (- 0.5)) (Math/sqrt 2.)))
-              z (* s (/ (- u1 0.5) u2))
-              zz (+ (* 0.25 z z) (Math/log u2))]
-          (if (> zz 0)
-            (recur mu sigma)
-            (+ mu (* sigma z)))))
-
-      (defn rand-normal []
-        (rand))
-
-      (defn rand-gaussian []
-        (rand-gaussian* 0 1.0))))
-
-(defn ms-elapsed
-  ([start]
-    (ms-elapsed start (timestamp)))
-  ([start end]
-    (let [start (double start)
-          end (double end)]
-      #?(:clj  (/ (- end start) 1000000.0))
-         :cljs (- end start))))
+;;;; Mathematics

 (defn tanh'
   "Compute the derivative of the tanh function for a given output.  Works on any array shape.
@@ -78,6 +33,49 @@
   [y]
   (m/emul y (m/sub 1.0 y)))

+;;;; Random number generation
+
+#?(:clj
+   (do
+     (def ^Random RAND-GENERATOR (Random.))
+
+     (defn rand-normal ^double []
+       (.nextDouble RAND-GENERATOR))
+
+     (defn rand-gaussian ^double []
+       (.nextGaussian RAND-GENERATOR)))
+   :cljs
+   (do
+     (defn rand-gaussian* [mu sigma]
+       ;; This function implements the Kinderman-Monahan ratio method:
+       ;;  A.J. Kinderman & J.F. Monahan
+       ;;  Computer Generation of Random Variables Using the Ratio of Uniform Deviates
+       ;;  ACM Transactions on Mathematical Software 3(3) 257-260, 1977
+       (let [u1  (rand)
+             u2* (rand)
+             u2 (- 1. u2*)
+             s (* 4 (/ (Math/exp (- 0.5)) (Math/sqrt 2.)))
+             z (* s (/ (- u1 0.5) u2))
+             zz (+ (* 0.25 z z) (Math/log u2))]
+         (if (> zz 0)
+           (recur mu sigma)
+           (+ mu (* sigma z)))))
+
+     (defn rand-normal []
+       (rand))
+
+     (defn rand-gaussian []
+       (rand-gaussian* 0 1.0))))
+
+;;;; Arrays and matrices
+
+(def EMPTY-VECTOR (m/new-array [0]))
+
+(defn empty-array
+  "Constructs a new empty (zero-filled) array of the given shape"
+  ([shape]
+   (m/new-array :vectorz shape)))
+
 (defn weight-matrix
   "Creates a randomised weight matrix.

@@ -86,30 +84,106 @@
   [rows cols]
   (let [weight-scale (Math/sqrt (/ 1.0 (double cols)))]
     (b/array
-     (mapv (fn [_]
-             (vec (repeatedly cols
-                              #(* weight-scale
-                                  (rand-gaussian)))))
-           (range rows)))))
+      (mapv (fn [_]
+              (vec (repeatedly cols
+                               #(* weight-scale
+                                   (rand-gaussian)))))
+            (range rows)))))

 (defn random-matrix
   "Constructs an array of the given shape with random normally distributed element values"
   ([shape-vector]
-    (rand-matrix/sample-normal shape-vector)))
-
-(defn empty-array
-  "Constructs a new empty (zero-filled) array of the given shape"
-  ([shape]
-    (m/new-array :vectorz shape)))
-
-(defn mse-gradient-fn
-  "Returns the MSE error gradient for a given output and target value"
-  ([output target]
-    (let [result (m/mutable output)]
-      (m/sub! result target)
-      (m/scale! result 2.0)
-      result)))
-
+   (rand-matrix/sample-normal shape-vector)))
+
+(defn assign-sparse-to-packed!
+  [packed-data sparse-data]
+  (let [packed-data (if packed-data
+                      packed-data
+                      (let [elem-count (reduce + (map m/ecount sparse-data))]
+                        (b/new-array [elem-count])))]
+    (reduce (fn [^long offset next-item]
+              (let [item-vec (m/as-vector next-item)
+                    item-len (long (m/ecount item-vec))]
+                (m/assign! (m/subvector packed-data offset item-len) item-vec)
+                (+ offset item-len)))
+            0
+            sparse-data)
+    packed-data))
+
+(defn assign-packed-to-sparse!
+  [sparse packed]
+  (reduce (fn [^long offset next-item]
+            (let [elem-count (long (m/ecount next-item))]
+              (m/assign! (m/as-vector next-item)
+                         (m/subvector packed offset elem-count))
+              (+ offset elem-count)))
+          0
+          sparse))
+
+(defn zero-sparse!
+  [sparse]
+  (doseq [item sparse]
+    (m/fill! item 0.0)))
+
+
+(defn get-or-new-array
+  [item kywd shape]
+  (or (get item kywd)
+      (b/new-array shape)))
+
+(defn get-or-array
+  [item kywd data]
+  (or (get item kywd)
+      (b/array data)))
+
+;;;; Confusion matrices
+
+(defn confusion-matrix
+  "A confusion matrix shows the predicted classes for each of the actual
+  classes in order to understand performance and commonly confused classes.
+
+                   Predicted
+                 Cat Dog Rabbit
+         | Cat	   5  3  0
+  Actual | Dog	   2  3  1
+         | Rabbit  0  2  11
+
+  Initialize with a set of string labels, and then call add-prediction for
+  each prediction to accumulate into the matrix."
+  [labels]
+  (let [prediction-map (zipmap labels (repeat 0))]
+    (into {} (for [label labels]
+               [label prediction-map]))))
+
+(defn add-prediction
+  [conf-mat prediction label]
+  (update-in conf-mat [label prediction] inc))
+
+(defn print-confusion-matrix
+  [conf-mat]
+  (let [ks (sort (keys conf-mat))
+        label-len (inc (int (apply max (map count ks))))
+        prefix (apply str (repeat label-len " "))
+        s-fmt (str "%" label-len "s")]
+    (apply println prefix ks)
+    (doseq [k ks]
+      (apply println (format s-fmt k) (map #(get-in conf-mat [k %]) ks)))))
+
+;;;; Time
+
+#?(:clj (defn timestamp [] (System/nanoTime))
+   :cljs (defn timestamp [] (.getTime (js/Date.))))
+
+(defn ms-elapsed
+  ([start]
+   (ms-elapsed start (timestamp)))
+  ([start end]
+   (let [start (double start)
+         end (double end)]
+     #?(:clj  (/ (- end start) 1000000.0))
+     :cljs (- end start))))
+
+;;;; Exceptions

 #?(:clj
    (defmacro error
@@ -124,49 +198,28 @@
 (defmacro error?
   "Returns true if executing body throws an error, false otherwise."
   ([& body]
-    `(try
-       ~@body
-       false
-       (catch Throwable t#
-         true))))
-
-(defn confusion-matrix
-  "A confusion matrix shows the predicted classes for each of the actual
-  classes in order to understand performance and commonly confused classes.
-
-                   Predicted
-                 Cat Dog Rabbit
-         | Cat	   5  3  0
-  Actual | Dog	   2  3  1
-         | Rabbit  0  2  11
-
-  Initialize with a set of string labels, and then call add-prediction for
-  each prediction to accumulate into the matrix."
-  [labels]
-  (let [prediction-map (zipmap labels (repeat 0))]
-    (into {} (for [label labels]
-               [label prediction-map]))))
-
-(defn add-prediction
-  [conf-mat prediction label]
-  (update-in conf-mat [label prediction] inc))
-
-(defn print-confusion-matrix
-  [conf-mat]
-  (let [ks (sort (keys conf-mat))
-        label-len (inc (int (apply max (map count ks))))
-        prefix (apply str (repeat label-len " "))
-        s-fmt (str "%" label-len "s")]
-    (apply println prefix ks)
-    (doseq [k ks]
-      (apply println (format s-fmt k) (map #(get-in conf-mat [k %]) ks)))))
+   `(try
+      ~@body
+      false
+      (catch Throwable t#
+        true))))
+
+;;;; Neural networks
+
+(defn mse-gradient-fn
+  "Returns the MSE error gradient for a given output and target value"
+  ([output target]
+   (let [result (m/mutable output)]
+     (m/sub! result target)
+     (m/scale! result 2.0)
+     result)))

 (def DIFFERENCE-DELTA 1e-5)

-; TODO: This will return the relative differences between the components of the
-; analytic gradient computed by the network and the numerical gradient, but it
-; would be nice if it just said, yes!  Need to figure out how close they should
-; be.  Maybe just check that each difference is < 0.01 ???
+;; TODO: This will return the relative differences between the components of the
+;; analytic gradient computed by the network and the numerical gradient, but it
+;; would be nice if it just said, yes!  Need to figure out how close they should
+;; be.  Maybe just check that each difference is < 0.01 ???
 (defn gradient-check
   "Use a finite difference approximation of the derivative to verify
   that backpropagation and the derivatives of layers and activation functions are correct.
@@ -202,50 +255,7 @@
           relative-error))
       (range (m/column-count input)))))

-
 (defn estimate-gradient
   "Computes a numerical approximation of the derivative of the function f at with respectr to input x"
   ([f x]
    (let [x+dx (m/add )])))
-
-
-(defn assign-sparse-to-packed!
-  [packed-data sparse-data]
-  (let [packed-data (if packed-data
-                      packed-data
-                      (let [elem-count (reduce + (map m/ecount sparse-data))]
-                        (b/new-array [elem-count])))]
-    (reduce (fn [^long offset next-item]
-              (let [item-vec (m/as-vector next-item)
-                    item-len (long (m/ecount item-vec))]
-                (m/assign! (m/subvector packed-data offset item-len) item-vec)
-                (+ offset item-len)))
-            0
-            sparse-data)
-    packed-data))
-
-(defn assign-packed-to-sparse!
-  [sparse packed]
-  (reduce (fn [^long offset next-item]
-            (let [elem-count (long (m/ecount next-item))]
-             (m/assign! (m/as-vector next-item)
-                        (m/subvector packed offset elem-count))
-             (+ offset elem-count)))
-          0
-          sparse))
-
-(defn zero-sparse!
-  [sparse]
-  (doseq [item sparse]
-    (m/fill! item 0.0)))
-
-
-(defn get-or-new-array
-  [item kywd shape]
-  (or (get item kywd)
-      (b/new-array shape)))
-
-(defn get-or-array
-  [item kywd data]
-  (or (get item kywd)
-      (b/array data)))


fix typo

fix typo

--- src/name_bazaar/ui/components/offering/middle_section.cljs
+++ src/name_bazaar/ui/components/offering/middle_section.cljs
@@ -52,7 +52,7 @@
                   " You'd need to send at least " min-bid-str "."]

                  :else
-                 [:div "You need to send at least " min-bid-str " to become the higest bidder."])))
+                 [:div "You need to send at least " min-bid-str " to become the highest bidder."])))

            (when active?
              [:div "To place a bid, you can send Ether directly into the " [d0x-misc/etherscan-link


Only render .build-actions when there are any

--- src-cljs/frontend/components/builds_table.cljs
+++ src-cljs/frontend/components/builds_table.cljs
@@ -151,20 +151,19 @@
            :title (build-model/status-words build)}
        (build-status-badge build)]

-      (when show-actions?
+      (when (build-model/can-cancel? build)
         [:div.build-actions
-         (when (build-model/can-cancel? build)
-           (let [build-id (build-model/id build)
-                 vcs-url (:vcs_url build)
-                 build-num (:build_num build)]
-             (forms/managed-button
-              [:button.cancel-build
-               {:data-loading-text "Canceling..."
-                :on-click #(raise! owner [:cancel-build-clicked {:build-id build-id
-                                                                 :vcs-url vcs-url
-                                                                 :build-num build-num}])}
-               [:img.cancel-icon {:src (common/icon-path "Status-Canceled")}]
-               [:span.cancel-text "Cancel"]])))])]
+         (let [build-id (build-model/id build)
+               vcs-url (:vcs_url build)
+               build-num (:build_num build)]
+           (forms/managed-button
+            [:button.cancel-build
+             {:data-loading-text "Canceling..."
+              :on-click #(raise! owner [:cancel-build-clicked {:build-id build-id
+                                                               :vcs-url vcs-url
+                                                               :build-num build-num}])}
+             [:img.cancel-icon {:src (common/icon-path "Status-Canceled")}]
+             [:span.cancel-text "Cancel"]]))])]

      [:div.build-info
       [:div.build-info-header


Show result at bottom of page.

Show result at bottom of page.

--- frontend/src/cljs/frontend/core.cljs
+++ frontend/src/cljs/frontend/core.cljs
@@ -123,8 +123,9 @@
        [:p "Inputs" [:br] [atom-textarea inputs]]
        [:button {:on-click (partial evaluate @inputs)}
         "Evaluate Inputs"]
-       [:code (:layer-text @app-state) (str (:network @app-state))]
-       [viz/draw {} (get-state :network)]])))
+       [viz/draw {} (get-state :network)]
+       [:code {:class (if (get-state :result) "visible" "hidden")}
+        (str (:result @app-state))]])))
 ;; -------------------------
 ;; Initialize app


Complete creating functions koans test

Complete creating functions koans test

--- clojure/koans/src/koans/12_creating_functions.clj
+++ clojure/koans/src/koans/12_creating_functions.clj
@@ -2,31 +2,31 @@

 (meditations
   "One may know what they seek by knowing what they do not seek"
-  (= [__ __ __] (let [not-a-symbol? (complement symbol?)]
+  (= [true false true] (let [not-a-symbol? (complement symbol?)]
                   (map not-a-symbol? [:a 'b "c"])))

   "Praise and 'complement' may help you separate the wheat from the chaff"
   (= [:wheat "wheat" 'wheat]
-       (let [not-nil? ___]
+       (let [not-nil? (complement nil?)]
          (filter not-nil? [nil :wheat nil "wheat" nil 'wheat nil])))

   "Partial functions allow procrastination"
   (= 20 (let [multiply-by-5 (partial * 5)]
-          (___ __)))
+          (multiply-by-5 4)))

   "Don't forget: first things first"
-  (= [__ __ __ __]
+  (= [:a :b :c :d]
        (let [ab-adder (partial concat [:a :b])]
-         (ab-adder [__ __])))
+         (ab-adder [:c :d])))

   "Functions can join forces as one 'composed' function"
   (= 25 (let [inc-and-square (comp square inc)]
-          (inc-and-square __)))
+          (inc-and-square 4)))

   "Have a go on a double dec-er"
-  (= __ (let [double-dec (comp dec dec)]
+  (= 8 (let [double-dec (comp dec dec)]
           (double-dec 10)))

   "Be careful about the order in which you mix your functions"
-  (= 99 (let [square-and-dec ___]
+  (= 99 (let [square-and-dec (comp dec square)]
           (square-and-dec 10))))


reduces info logging on the expected cache removal (#488)

--- waiter/src/waiter/util/cache_utils.clj
+++ waiter/src/waiter/util/cache_utils.clj
@@ -29,9 +29,9 @@
         ttl (.expireAfterWrite ttl TimeUnit/MILLISECONDS))
       (.removalListener (proxy [RemovalListener] []
                           (onRemoval [^RemovalNotification n]
-                            (log/info "cache entry removal notification"
-                                      {:evicted (.wasEvicted n)
-                                       :key (.getKey n)}))))
+                            (log/debug "cache entry removal notification"
+                                       {:evicted (.wasEvicted n)
+                                        :key (.getKey n)}))))
       (.build)))

 (defn cache-contains?


Clean up my botched rebase that mixed up delay and @ usage

--- web/src/immutant/web/undertow/http.clj
+++ web/src/immutant/web/undertow/http.clj
@@ -51,8 +51,8 @@
     (.getHeaderNames headers)))

 (defn- ring-request [^HttpServerExchange exchange]
-  (let [headers (delay (.getRequestHeaders exchange))
-        content-type (delay (.getFirst ^HeaderMap @headers Headers/CONTENT_TYPE))]
+  (let [headers (.getRequestHeaders exchange)
+        content-type (delay (.getFirst ^HeaderMap headers Headers/CONTENT_TYPE))]
     ;; TODO: context, path-info ?
     (LazyMap. {:server-port (delay (-> exchange .getDestinationAddress .getPort))
                :server-name (delay (.getHostName exchange))
@@ -61,10 +61,10 @@
                :query-string (delay (.getQueryString exchange))
                :scheme (delay (-> exchange .getRequestScheme keyword))
                :request-method (delay (-> exchange .getRequestMethod .toString .toLowerCase keyword))
-               :content-type content-type
+               :content-type (delay @content-type)
                :content-length (delay (.getRequestContentLength exchange))
-               :character-encoding (delay (if content-type
-                                            (Headers/extractTokenFromHeader content-type "charset")))
+               :character-encoding (delay (if @content-type
+                                            (Headers/extractTokenFromHeader @content-type "charset")))
                ;; TODO: :ssl-client-cert
                :headers (delay (headers->map headers))
                :body (delay (.getInputStream exchange))})))


scheduler:  Use a newer, forked version of Mesomatic.

--- scheduler/project.clj
+++ scheduler/project.clj
@@ -86,7 +86,7 @@

                  ;;External system integrations
                  [me.raynes/conch "0.5.2"]
-                 [spootnik/mesomatic "1.0.1-r0"]
+                 [wyegelwe/mesomatic "1.0.1-r0-SNAPSHOT"]
                  [org.clojure/tools.nrepl "0.2.3"]

                  ;;Ring


Sample web UI: use string enums in JSON requests

Reviewers: zarko, shahms, fromberger

Reviewed By: fromberger

Differential Revision: https://phabricator-dot-kythe-repo.appspot.com/D977

--- kythe/web/ui/src-cljs/ui/service.cljs
+++ kythe/web/ui/src-cljs/ui/service.cljs
@@ -75,10 +75,10 @@
    (get-xrefs ticket {} handler error-handler))
   ([ticket opts handler error-handler]
    (POST "xrefs"
-     {:params (merge {:definition_kind 3    ;; BINDING_DEFINITIONS
-                      :declaration_kind 1   ;; ALL_DECLARATIONS
-                      :reference_kind 1     ;; ALL_REFERENCES
-                      :documentation_kind 1 ;; ALL_DOCUMENTATION
+     {:params (merge {:definition_kind    "BINDING_DEFINITIONS"
+                      :declaration_kind   "ALL_DECLARATIONS"
+                      :reference_kind     "ALL_REFERENCES"
+                      :documentation_kind "ALL_DOCUMENTATION"
                       :filter [schema/node-kind-fact]
                       :anchor_text true
                       :page_size 20}


CMR-1885 Typo for SERF

CMR-1885 Typo for SERF

--- umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/serf.clj
+++ umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/serf.clj
@@ -59,7 +59,7 @@

 (defn parse-serf-xml
-  "Returns collection map from DIF10 collection XML document."
+  "Returns collection map from a SERF XML document."
   [doc]
   {:EntryTitle (value-of doc "/SERF/Entry_Title")
    :EntryId (value-of doc "/SERF/Entry_ID")


:bug: Fix text rendering on svg export.

:bug: Fix text rendering on svg export.

--- exporter/src/app/http/export_svg.cljs
+++ exporter/src/app/http/export_svg.cljs
@@ -84,7 +84,7 @@
   (letfn [(render-in-page [page {:keys [uri cookie] :as rctx}]
             (p/do!
              (bwr/emulate! page {:viewport [1920 1080]
-                                 :scale 2})
+                                 :scale 4})
              (bwr/set-cookie! page cookie)
              (bwr/navigate! page uri)
              (bwr/eval! page (js* "() => document.body.style.background = 'transparent'"))
@@ -227,7 +227,7 @@

           (render-svg [page]
             (p/let [dom   (bwr/select page "#screenshot")
-                    main  (bwr/eval! dom (fn [elem] (.-innerHTML ^js elem)))
+                    main  (bwr/eval! dom (fn [elem] (.-outerHTML ^js elem)))
                     nodes (process-text-nodes page)]
               (replace-nodes-on-main main nodes)))


rename garden site target to prod

rename garden site target to prod

--- src/leiningen/new/reagent_liveedit/project.clj
+++ src/leiningen/new/reagent_liveedit/project.clj
@@ -58,14 +58,14 @@
             "web"      ["with-profile" "production" "trampoline" "ring" "server"]
             "prod"     ["with-profile" "production" "do"
                         "clean,"
-                        "garden" "once" "site,"
+                        "garden" "once" "prod,"
                         "cljsbuild" "once" "app"]
             "live"     ["pdo" "css," "figwheel," "server"]
             "once"     ["do" "cljsbuild" "once" "app,"
                         "garden" "once"]
             "dev"      ["do" "once," "live"]}

-  :garden {:builds [{:id "site"
+  :garden {:builds [{:id "prod"
                      :source-paths ["src/styles"]
                      :stylesheet {{ns-name}}.styles.site/site
                      :compiler {:output-to "resources/public/css/site.css"


expand ["SEGMENT" <segment-id>] clauses in the QP.

--- src/metabase/driver/query_processor/expand.clj
+++ src/metabase/driver/query_processor/expand.clj
@@ -14,7 +14,8 @@
             [metabase.util :as u])
   (:import (clojure.lang Keyword)))

-(declare parse-aggregation
+(declare expand-macros
+         parse-aggregation
          parse-breakout
          parse-fields
          parse-filter
@@ -45,15 +46,19 @@
     (binding [*original-query-dict* query-dict]
       ;; TODO - we should parse the Page clause so we can validate it
       ;; And convert to a limit / offset clauses
-      (update query-dict :query #(-<> (assoc %
-                                             :aggregation (parse-aggregation (:aggregation %))
-                                             :breakout    (parse-breakout    (:breakout %))
-                                             :fields      (parse-fields      (:fields %))
-                                             :filter      (parse-filter      (:filter %))
-                                             :order_by    (parse-order-by    (:order_by %)))
-                                      (set/rename-keys <> {:order_by     :order-by
-                                                           :source_table :source-table})
-                                      (m/filter-vals non-empty-clause? <>))))))
+      (-> query-dict
+          ;; expand/resolve macro type clauses first (metrics & segments)
+          expand-macros
+          ;; now parse the base clauses, fields, & values
+          (update :query #(-<> (assoc %
+                                 :aggregation (parse-aggregation (:aggregation %))
+                                 :breakout    (parse-breakout    (:breakout %))
+                                 :fields      (parse-fields      (:fields %))
+                                 :filter      (parse-filter      (:filter %))
+                                 :order_by    (parse-order-by    (:order_by %)))
+                               (set/rename-keys <> {:order_by     :order-by
+                                                    :source_table :source-table})
+                               (m/filter-vals non-empty-clause? <>)))))))

 ;; ## -------------------- Field + Value --------------------
@@ -267,3 +272,27 @@
                                                                       (parse-order-by-direction direction)))
 (defparser parse-order-by
   subclauses (mapv parse-order-by-subclause subclauses))
+
+
+;; # ======================================== MACRO EXPANSION ========================================
+
+
+(defparser segment-parse-filter-subclause
+  ["SEGMENT" (segment-id :guard integer?)] [(-> (sel :one :field [metabase.models.segment/Segment :definition] :id segment-id)
+                                                :filter)]
+  subclause  [subclause])
+
+(defparser segment-parse-filter
+  ["AND" & subclauses] (vec (apply concat ["AND"] (mapv segment-parse-filter subclauses)))
+  ["OR" & subclauses]  (vec (apply concat ["OR"] (mapv segment-parse-filter subclauses)))
+  subclause            (segment-parse-filter-subclause subclause))
+
+(defn- macroexpand-segment
+  [query-dict]
+  (if (non-empty-clause? (get-in query-dict [:query :filter]))
+    (update-in query-dict [:query :filter] segment-parse-filter)
+    query-dict))
+
+(defn expand-macros "Expand the macros (Segment, Metric) in a QUERY-DICT."
+  [query-dict]
+  (macroexpand-segment query-dict))


[SD] Don't forget critical errors

[SD] Don't forget critical errors

--- test/vip/data_processor/validation/xml_with_postgres_test.clj
+++ test/vip/data_processor/validation/xml_with_postgres_test.clj
@@ -57,6 +57,7 @@

   (testing "when there are no errors, the errors table doesn't grow"
     (let [ctx {:fatal {}
+               :critical {}
                :warnings {:import
                           {:global
                            {:invalid-extensions []}}}


Reorder some args

Reorder some args

--- src/angelic/search/function_sets.clj
+++ src/angelic/search/function_sets.clj
@@ -19,12 +19,12 @@

 (defprotocol FunctionSet
   (fs-name    [fs]           "Arbitrary name to identify fs")
+  (precondition-context-set [fs input-set] "Relevant set of variables")
   (apply-opt  [fs input-set] "[output-set upper-reward-bound status]. output nil if empty or -inf. Status is :live/:blocked/:solved.")
   (apply-pess [fs input-set] "[output-set upper-reward-bound status]. output nil if empty or -inf. Status is :live/:blocked/:solved.")
   (child-seqs [fs input-set] "seq of seqs of FunctionSets. Only valid if above is :live.
                               (= :live (status fs s)) ==>
-                                 (subset? (child-seqs fs s-subset) (child-seqs fs s)) (for names)")
-  (precondition-context-set [fs input-set] "Relevant set of variables"))
+                                 (subset? (child-seqs fs s-subset) (child-seqs fs s)) (for names)"))

 (defn extract-context [fs input-set]
   (state/extract-context input-set (precondition-context-set fs input-set)))
@@ -48,10 +48,10 @@
                             (= n (fs-name ofs))))
     FunctionSet
     (fs-name    [fs]           n)
+    (precondition-context-set [fs input-set] (angelic/precondition-context-set action input-set))
     (apply-opt  [fs input-set] (output-or-nil angelic/optimistic-set-and-reward status-fn action input-set))
     (apply-pess [fs input-set] (output-or-nil angelic/pessimistic-set-and-reward status-fn action input-set))
-    (child-seqs [fs input-set] (child-seq-fn input-set))
-    (precondition-context-set [fs input-set] (angelic/precondition-context-set action input-set)))))
+    (child-seqs [fs input-set] (child-seq-fn input-set)))))

 (defmethod print-method angelic.search.function_sets.FunctionSet [s o]
            (print-method (fs-name s) o))


formatting

formatting

--- test/clj/backtype/storm/transactional_test.clj
+++ test/clj/backtype/storm/transactional_test.clj
@@ -25,8 +25,6 @@
 ;;  should test that it commits even when receiving no tuples (and test that finishBatch is called before commit in this case)

-;; * Test that it repeats the meta for a partitioned state (test partitioned emitter on its own)
-;; * Test that partitioned state emits nothing for the partition if it has seen a future transaction for that partition (test partitioned emitter on its own)

 (defn mk-coordinator-state-changer [atom]
   (TransactionalSpoutCoordinator.
@@ -273,8 +271,6 @@
 (defn- cleanup-before [state txid]
   (.cleanupBefore state (to-txid txid)))

-
-
 (deftest test-rotating-transactional-state
   ;; test strict ordered vs not strict ordered
   (let [zk-port (available-port 2181)]
@@ -316,5 +312,5 @@
         (.close state)
         ))))

-;; * Test that coordinator and partitioned state are cleaned up properly (and not too early) - test rotatingtransactionalstate
-
+;; * Test that it repeats the meta for a partitioned state (test partitioned emitter on its own)
+;; * Test that partitioned state emits nothing for the partition if it has seen a future transaction for that partition (test partitioned emitter on its own)


progress on transactional tests, working so far

--- test/clj/backtype/storm/transactional_test.clj
+++ test/clj/backtype/storm/transactional_test.clj
@@ -53,7 +53,7 @@
         ))))

 (defn normalize-tx-tuple [values]
-  (-> values vec (update 0 (memfn getTransactionId))))
+  (-> values vec (update 0 #(-> % .getTransactionId .intValue))))

 (defn verify-and-reset! [expected-map emitted-map-atom]
   (let [results @emitted-map-atom]
@@ -391,20 +391,19 @@
                         (.createTopology builder)))
        (submit-local-topology (:nimbus cluster)
                               "transactional-test"
-                              {TOPOLOGY-MAX-SPOUT-PENDING 2
-                               TOPOLOGY-DEBUG true}
+                              {TOPOLOGY-MAX-SPOUT-PENDING 2}
                               (:topology topo-info))

-       (bind ack-tx (fn [txid]
-                      (let [[to-ack not-to-ack] (separate
-                                                 #(-> %
-                                                      (.getValue 0)
-                                                      .getTransactionId
-                                                      (= txid))
-                                                 @tuples)]
-                        (reset! tuples not-to-ack)
-                        (doseq [t to-ack]
-                          (.ack @collector t)))))
+       (bind ack-tx! (fn [txid]
+                       (let [[to-ack not-to-ack] (separate
+                                                  #(-> %
+                                                       (.getValue 0)
+                                                       .getTransactionId
+                                                       (= txid))
+                                                  @tuples)]
+                         (reset! tuples not-to-ack)
+                         (doseq [t to-ack]
+                           (.ack @collector t)))))

        ;; only check default streams
        (bind verify! (fn [expected]
@@ -421,10 +420,28 @@
        (tracked-wait topo-info 2)
        (println "Controlled: " @tuples)
        (println "Captured:" (-> topo-info :capturer .getResults))
-       ;; check that batch tuples have been emitted for tx 1 and 2
-       ;; check the outputs of all the bolts
-       ;; ack the ack the first batch
-       ;; check that first batch commits
+       (verify! {"sum" [[1 "dog" 3]
+                        [1 "cat" 5]
+                        [1 "mango" 6]
+                        [1 "happy" 11]
+                        [2 "apple" 1]
+                        [2 "dog" 3]
+                        [2 "zebra" 1]]
+                 "count" []
+                 "count2" []})
+       (ack-tx! 1)
+       (tracked-wait topo-info 1)
+       (verify! {"sum" []
+                 "count" [[1 "dog" 1]
+                          [1 "cat" 2]
+                          [1 "mango" 2]
+                          [1 "happy" 1]]
+                 "count2" [[1 "dog" 2]
+                           [1 "cat" 2]
+                           [1 "mango" 2]
+                           [1 "happy" 2]]})
+       (ack-tx! 1)
+
        ;; ack the commit
        ;; check that third batch is emitted
        ;; ack the third batch
@@ -440,3 +457,14 @@
        ;;       (println (read-tuples results "count"))
        (-> topo-info :capturer .getAndClearResults)
        ))))
+
+{0 [["dog" 3]
+    ["cat" 4]
+    ["apple" 1]
+    ["dog" 3]
+    ["banana" 0]]
+ 1 [["cat" 1]
+    ["mango" 4]]
+ 2 [["happy" 11]
+    ["mango" 2]
+    ["zebra" 1]]}


added tag to defrule

added tag to defrule

--- libs/cloudlog/test/cloudlog/core_test.clj
+++ libs/cloudlog/test/cloudlog/core_test.clj
@@ -2,7 +2,7 @@
   (:use midje.sweet)
   (:use [cloudlog.core]))

-[[:chapter {:title "defrule: Rule Definition Macro"}]]
+[[:chapter {:title "defrule: Rule Definition Macro" :tag "defrule"}]]
 "Definition:"
 [[:reference {:refer "cloudlog.core/defrule"}]]


cofigure for lift run

cofigure for lift run

--- src/hydrofoil/core.clj
+++ src/hydrofoil/core.clj
@@ -39,6 +39,7 @@

         (do (print-content-csv performance-data)
             (println "")
+            (flush)
             parent)

         (if (> child-score parent-score)
@@ -46,6 +47,7 @@
           (if (= (mod num-tries 250) 0)
             (do (print-content-csv performance-data)
                 (println "")
+                (flush)
                 (recur child (inc num-tries) '() ))
             (recur child (inc num-tries) (conj performance-data (conj parent {:lift parent-score :cl cl}))))

@@ -58,11 +60,16 @@

         )))))

-(defn -main [& args]
-  (hill-climber 100 (run-constants 1 50 -10)))
+(defn -main [angle-of-attack]
+  ;(println (inc  (Integer/parseInt angle-of-attack))))
+  (hill-climber 1000000 (run-constants 1 50 (Integer/parseInt angle-of-attack))))

+; NACA 2412 in air at 30 m/s with 0 degrees aot: cl = 0.22745, lift = 10.24945
+; NACA 2412 in air at 30 m/s with 4 degrees aot: cl = 0.66611, lift = 30.01527
+; NACA 2412 in air at 30 m/s with 8 degrees aot: cl = 1.10476, lift = 49.78110
+; NACA 2412 in air at 30 m/s with 12 degrees aot: cl = 1.543406, lift = 69.54692
+; NACA 2412 in air at 30 m/s with 16 degrees aot: cl = 1.982055, lift = 89.31274
+(lift-function (NACA-design 2 4 12) (run-constants 1.225 30 16))

-
-


Be sure to fetch --all remote refs.

Be sure to fetch --all remote refs.

--- src/leiningen/voom.clj
+++ src/leiningen/voom.clj
@@ -181,7 +181,7 @@

 (defn dirty-repo?
   [gitdir]
-  (git {:gitdir gitdir} "fetch")
+  (git {:gitdir gitdir} "fetch" "--all")
   (let [g {:gitdir gitdir}
         dirty (:lines (git g "status" "--short"))
         stashes (:lines (git g "stash" "list"))
@@ -242,7 +242,7 @@
        (let [repo (-> (remotes d) :origin :fetch)]
          (println "Fetching:" repo (print-str (list (.getName d))))
          (try
-           (git {:gitdir d} "fetch")
+           (git {:gitdir d} "fetch" "--all")
            (catch clojure.lang.ExceptionInfo e
              (let [data (ex-data e)]
                (if (:git data)
@@ -677,7 +677,7 @@
                  (not= repo (-> (remotes (:gitdir g)) :origin :fetch)))
         (safe-delete-repo checkout pdir))
       (if (.exists ^File checkout)
-        (git g "fetch")
+        (git g "fetch" "--all")
         (git {} "clone" repo "--refer" gitdir checkout))
       (safe-checkout checkout sha) ; must detach head for update...
       (git g "branch" "-f" branch sha)


Improve docstrings so they render better in API docs

--- src/re_frame/core.cljc
+++ src/re_frame/core.cljc
@@ -312,20 +312,20 @@
     - `handler` is a side-effecting function which takes a single argument and whose return
       value is ignored.

-   To use, first, associate `:effect2` with a handler.
+  To use, first, associate `:effect2` with a handler:

       (reg-fx
          :effect2
          (fn [value]
             ... do something side-effect-y))

-   Then, later, if an event handler were to return this effects map ...
+  Then, later, if an event handler were to return this effects map:

       {:effect2  [1 2]}

-   then the `handler` `fn` we registered previously, using `reg-fx`, will be
-   called with an argument of `[1 2]`.
-   "
+  then the `handler` `fn` we registered previously, using `reg-fx`, will be
+  called with an argument of `[1 2]`.
+  "
   [id handler]
   (fx/reg-fx id handler))

@@ -715,6 +715,7 @@
                    ... modifies and returns `context`)))

   Notes:
+
     - `:before` functions modify and return their `context` argument. Unless they
       side effect, in which case, they'll perform the side effect and return
       `context` unchanged.
@@ -792,7 +793,7 @@
 (defn set-loggers!
   "re-frame outputs warnings and errors via the API function `console`
    which, by default, delegates to `js/console`'s default implementation for
-  `log, `error`, `warn` `debug` `group` `groupEnd`. But, using this function,
+  `log`, `error`, `warn`, `debug`, `group` and `groupEnd`. But, using this function,
    you can override that behaviour with your own functions.

   The argument `new-loggers` should be a map containing a subset of they keys
@@ -820,11 +821,11 @@
   By default, it will output the given `args` to `js/console` at the given log `level`.
   However, an application using re-frame can redirect `console` output via `set-loggers!`.

-  `level` can be one of `:log` `:error` `:warn` `:debug` `:group` `:groupEnd`.
+  `level` can be one of `:log`, `:error`, `:warn`, `:debug`, `:group` or `:groupEnd`.

   Example usage:

-      (console :error \"Oh, dear God, it happened:\" a-var \"and\" another)
+      (console :error \"Sure enough it happened:\" a-var \"and\" another)
       (console :warn \"Possible breach of containment wall at:\" dt)
   "
   [level & args]


Bug fix for undo.cljs

Bug fix for undo.cljs

--- src/re_frame/undo.cljs
+++ src/re_frame/undo.cljs
@@ -227,24 +227,5 @@
     (fn fsm-handler
       [db event-v]
       (let [new-db (handler db event-v)]
-         (update-fn new-db event-v trigger fsm-path))))   ;; think about access to event-v
-
-
-(register-handler
-  :database-connection
-  (fsm-tirgger :db-working bootstrap-fsm [:fsms :bnootstrap] )
-  (fn [db q]
-    .....)
-  )
-
-
-;; state
-;;    fms-state
-;;    seen-evetns
-;;    started-tasks
-
-(defn bootstrap-fsm
-  [new-db event-v trigger fsm-path]
-  (let [new-state ]
-    )
-  )
+         (update-fn new-db event-v trigger fsm-path)))))   ;; think about access to event-v
+


client: switch to sendWithAck

client: switch to sendWithAck

--- src/riemann/client.clj
+++ src/riemann/client.clj
@@ -67,7 +67,7 @@
       (when-let [t (:time event)] (.time e (int t)))
       (when-let [t (:ttl event)] (.ttl e (float t)))

-      (.send e))))
+      (.sendWithAck e))))

 (defn tcp-client
   "Create a new TCP client. Example:


Minor doc enhance

Minor doc enhance

--- messaging/src/immutant/messaging.clj
+++ messaging/src/immutant/messaging.clj
@@ -215,7 +215,8 @@
     (o/boolify :decode)))

 (defn listen
-  "Registers `f` to receive each message sent to `destination`.
+  "Registers a single-arity function `f` to handle messages published
+   to `destination`.

    If a :selector is provided, then only messages having
    metadata/properties matching that expression will be received.


CMR-1050: Addressing code review comments.

--- bootstrap-app/src/cmr/bootstrap/data/db_synchronization.clj
+++ bootstrap-app/src/cmr/bootstrap/data/db_synchronization.clj
@@ -11,6 +11,7 @@
             [cmr.bootstrap.data.migration-utils :as mu]
             [cmr.common.util :as util]
             [cmr.common.concepts :as concepts]
+            [cmr.common.config :as config]
             [clj-time.coerce :as cr]
             [clj-time.core :as t]
             [cmr.metadata-db.data.oracle.concept-tables :as tables]
@@ -22,15 +23,15 @@

 (def WORK_ITEMS_BATCH_SIZE
   "The number of work items to fetch at a time from the work items table during processing"
-  1000)
+  (config/config-value :db-sync-work-items-batch-size "1000" #(Long. ^String %)))

 (def NUM_PROCESS_INSERT_THREADS
   "The number of concurrent threads that should process items for insert."
-  5)
+  (config/config-value :db-sync-num-insert-threads "5" #(Long. ^String %)))

 (def NUM_PROCESS_DELETE_THREADS
   "The number of concurrent threads that should process deleting items"
-  5)
+  (config/config-value :db-sync-num-delete-threads "5" #(Long. ^String %)))

 (comment

@@ -52,7 +53,9 @@
         (sql-utils/query conn stmt)))

 (defn- add-missing-items-to-work-table
-  "Finds items that are missing in Metadata DB and adds it to the sync_work table."
+  "Finds all the items that were ingested between start-date and end-date in Catalog REST and adds
+  them to the sync_work table. These are all not necessarily missing from Metadata DB. We'll treat
+  them as if they were since we know ingest into Metadata DB is idempotent."
   [system provider-id concept-type start-date end-date]
   (truncate-work-table system)
   (let [sql (format "insert into sync_work (id, concept_id)
@@ -71,7 +74,7 @@

 (defn- in-clause
   "Generates a sql in clause string. If there are more than the max values to put in the in clause
-  it creates multiple an ANDs them together."
+  it creates multiple an ORs them together."
   [field num-values]
   (let [num-full (int (/ num-values 1000))
         num-in-partial (mod num-values 1000)


Fixing parameter converters attribute test.

--- test/cmr/search/test/services/parameter_converters/attribute.clj
+++ test/cmr/search/test/services/parameter_converters/attribute.clj
@@ -64,8 +64,10 @@

 (deftest parameter->condition-test
   (testing "single value condition"
-    (is (= [(qm/->AttributeValueCondition :string "alpha" "a")]
+    (is (= (qm/->AttributeValueCondition :string "alpha" "a")
            (p/parameter->condition :granule :attribute ["string,alpha,a"] {}))))
   (testing "single range condition"
-    (is (= [(qm/->AttributeRangeCondition :string "alpha" "a" "b")]
+    (is (= (qm/->AttributeRangeCondition :string "alpha" "a" "b")
            (p/parameter->condition :granule :attribute ["string,alpha,a,b"] {})))))
+
+;; TODO Add test for multiple


CMR-1855 Adds docstring.

CMR-1855 Adds docstring.

--- umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/iso19115_2.clj
+++ umm-spec-lib/src/cmr/umm_spec/umm_to_xml_mappings/iso19115_2.clj
@@ -32,6 +32,7 @@
                             :codeListValue date-name} date-name]]]])

 (defn- make-instrument-title
+  "Returns an ISO title string from a XPath context containing an instrument record."
   [{[instrument] :context}]
   (str (:ShortName instrument)
        ">"


Adds SpatialKeywords to new ECHO10 parsing.

--- umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/new_echo10.clj
+++ umm-spec-lib/src/cmr/umm_spec/xml_to_umm_mappings/new_echo10.clj
@@ -56,6 +56,7 @@
     :AccessConstraints {:Description (value-of doc "/Collection/RestrictionComment")
                         :Value (value-of doc "/Collection/RestrictionFlag")}
     :TemporalKeywords (values-at doc "/Collection/TemporalKeywords/Keyword")
+    :SpatialKeywords (values-at doc "/Collection/SpatialKeywords/Keyword")
     :TemporalExtents (parse-temporal doc)
     :Platforms (for [plat (select doc "/Collection/Platforms/Platform")]
                  (assoc (fields-from plat :ShortName :LongName :Type)


reverted added usage note

reverted added usage note

--- src/clj/forma/postprocess/output.clj
+++ src/clj/forma/postprocess/output.clj
@@ -59,8 +59,6 @@
   (reduce str (split date-str #"-")))

 (defn mk-date-header
-  "(mk-date-header \"16\" \"p\" (s/timeseries-value 827 (range 141)))
-  => (\"p20051219\" \"p20060101\" \"p20060117\" ... \"p20120117\")"
   [tres prefix ts]
   (map (comp (partial str prefix)
              date-no-sep


`find-replacement-lessons-within-time` - use destructuring for arg

Something we picked up from:

    http://blog.jayfields.com/2010/07/clojure-destructuring.html

after wondering if we could do what we did in this patch. The answer is
yes =)

We'll be making use of this technique a lot after the major refactorings
are completed.

Signed-off-by: Pang Yan Han <22594c345857f664c61cd411f607c4d40a9d4af9@gmail.com>

--- src/cljs/cljs_nusmods/timetable.cljs
+++ src/cljs/cljs_nusmods/timetable.cljs
@@ -1160,12 +1160,8 @@
    Returns a map, where keys are `TimetableLessonInfo` objects augmented with
    `:day` and `:rowNum` keys, and values are jQuery <div> objects for the
    shifted lesson."
-  [removedAugTTLessonInfo]
-  (let [day        (:day removedAugTTLessonInfo)
-        rowNum     (:rowNum removedAugTTLessonInfo)
-        startTime  (:startTime removedAugTTLessonInfo)
-        endTime    (:endTime removedAugTTLessonInfo)
-        ttDay      (timetable-get-day day)
+  [{:keys [day rowNum startTime endTime] :as removedAugTTLessonInfo}]
+  (let [ttDay (timetable-get-day day)

         ; This is the time range freed up by removal of the lesson
         [lowTimeIdx highTimeIdx]


[REFACTOR] Add `reset-ModulesSelectedOrder!` function

Signed-off-by: Pang Yan Han <22594c345857f664c61cd411f607c4d40a9d4af9@gmail.com>

--- src/cljs/cljs_nusmods/timetable.cljs
+++ src/cljs/cljs_nusmods/timetable.cljs
@@ -278,6 +278,11 @@
   "Removes a module from `ModulesSelectedOrder`"
   [moduleCode]
   (set! ModulesSelectedOrder (remove #{moduleCode} ModulesSelectedOrder)))
+
+(defn- reset-ModulesSelectedOrder!
+  "Resets `ModuleSelectedOrder`"
+  []
+  (set! ModulesSelectedOrder []))

 (defn- select2-box-update-modules
   "Update the select2 box with the currently selected modules"
@@ -1550,6 +1555,6 @@
       (attr (.find $dayElem "tr > th") "rowspan" 2)))
   (timetable-create!)
   (reset-ModulesSelected!)
-  (set! ModulesSelectedOrder [])
+  (reset-ModulesSelectedOrder!)
   (select2-box-update-modules)
   (set-document-location-hash! ""))


Selected option support.

Selected option support.

--- src/cljs/town_client/templating.cljs
+++ src/cljs/town_client/templating.cljs
@@ -35,17 +35,21 @@
      (render template params)))))

-(defn neighborhood-options [neighborhoods]
+(defn neighborhood-options [neighborhoods selected]
   (map (fn [n]
-         (node [:option {:value (:id n)} (:name n)]))
+         (node [:option
+                ^:attrs (if (= (:id n) (js/parseInt selected))
+                          {:value (:id n) :selected :selected}
+                          {:value (:id n)})
+                (:name n)]))
        neighborhoods))

 (defn choose-neighborhood [id]
   (aset (.-location js/window) "hash" id))

-(defn populate-neighborhood-menu [neighborhoods]
+(defn populate-neighborhood-menu [neighborhoods selected]
   (let [dropdown (sel1 :.navigate-areas)
-        options  (neighborhood-options neighborhoods)]
+        options  (neighborhood-options neighborhoods selected)]
     (dm/clear! dropdown)
     (doseq [no options]
       (dm/append! dropdown no))
@@ -104,14 +108,8 @@

 (defn reinit-page
   [user-channel]
-  (apply-template (sel1 ".page div") {} "kaupunginosa")
-  ;; (doseq [placeholder (sel "div[data-placeholder=\"true\"]")]
-  ;;     (apply-template placeholder {}))
-
-  )
+  (apply-template (sel1 ".page div") {} "kaupunginosa"))

 (defn init [channel]
-  (log "Init templating.")
   (add-event-handlers channel)
-  (apply-template (sel1 ".site-header div") {} "site-header")
-)
+  (apply-template (sel1 ".site-header div") {} "site-header"))


Track tot-puts/tot-takes in grouper.

Track tot-puts/tot-takes in grouper.

--- src/cbfg/grouper.cljs
+++ src/cbfg/grouper.cljs
@@ -13,7 +13,7 @@
    The grouper closes the take-all-ch after put-ch is closed
    and all the entries are taken."
   (act-loop grouper actx
-            [entries {} max-entries max-entries]
+            [entries {} max-entries max-entries tot-puts 0 tot-takes 0]
             (let [n (count entries)
                   chs0 (if (>= n max-entries)
                          []        ; We're full, so ignore put'ers.
@@ -23,10 +23,11 @@
                          chs0)]    ; We're empty, so ignore takers.
               (if (seq chs1)
                 (let [[m ch] (aalts grouper chs1)]
-                  (cond
-                   (= ch put-ch) (if-let [[k f] m]
-                                   (recur (update-in entries [k] f) max-entries)
-                                   (recur entries -1))
-                   (= ch take-all-ch) (recur {} max-entries)))
+                  (if (= ch put-ch)
+                    (if-let [[k f] m]
+                      (recur (update-in entries [k] f) max-entries
+                             (inc tot-puts) tot-takes)
+                      (recur entries -1 tot-puts tot-takes))
+                    (recur {} max-entries tot-puts (inc tot-takes))))
                 (aclose grouper take-all-ch)))))


correct theme text on some levels

correct theme text on some levels

--- src/client/game/core.cljs
+++ src/client/game/core.cljs
@@ -349,18 +349,18 @@
                      :three (do
                               (swap! state assoc :theme 2)
                               (.preventDefault e)
-                              (.html ($ "#theme") "1989")
-                              (.html ($ "#theme-details") "Gameboy"))
+                              (.html ($ "#theme") "1986")
+                              (.html ($ "#theme-details") "Tengen/Atari Arcade"))
                      :four (do
                              (swap! state assoc :theme 3)
                              (.preventDefault e)
                              (.html ($ "#theme") "1989")
-                             (.html ($ "#theme-details") "NES"))
+                             (.html ($ "#theme-details") "Gameboy"))
                      :five (do
                              (swap! state assoc :theme 4)
                              (.preventDefault e)
-                             (.html ($ "#theme") "1986")
-                             (.html ($ "#theme-details") "Tengen/Atari Arcade"))
+                             (.html ($ "#theme") "1989")
+                             (.html ($ "#theme-details") "NES"))
                      :six (do
                             (swap! state assoc :theme 5)
                             (.preventDefault e)


Start using cljs-ajax capabilities for forming GET query string.

--- metlog-vault/src-client/metlog_client/metlog.cljs
+++ metlog-vault/src-client/metlog_client/metlog.cljs
@@ -25,16 +25,22 @@
                                    (put! c x)))]))
     c))

-(defn ajax-get [ url callback ]
-  (ajax/GET url {:handler callback
-                 :error-handler (fn [ resp ]
-                                  (.log js/console (str "HTTP error, url: " url " resp: " resp)))}))
+(defn ajax-get
+  ([ url callback params ]
+   (ajax/GET url {:handler callback
+                  :params params
+                  :error-handler (fn [ resp ]
+                                   (.log js/console (str "HTTP error, url: " url " resp: " resp)))}))
+  ( [ url callback ]
+    (ajax-get url callback {})))

 (defn fetch-series-names [ cb ]
   (ajax-get "/series-names" cb))

 (defn fetch-series-data [ series-name query-window-secs cb ]
-  (ajax-get (str "/data/" series-name "?query-window-secs=" query-window-secs) cb))
+  (ajax-get (str "/data/" series-name)
+            cb
+            {:query-window-secs= query-window-secs}))

 (defn dom-node-width
   ([ node default-width ]


add @constructor annotation for deftypes, refs #4

--- src/clj/cljs/compiler.clj
+++ src/clj/cljs/compiler.clj
@@ -347,6 +347,7 @@
 (defmethod emit :deftype*
   [{:keys [t fields]}]
   (let [fields (map munge fields)]
+    (println "\n/**\n* @constructor\n*/")
     (println (str t " = (function (" (comma-sep (map str fields)) "){"))
     (doseq [fld fields]
       (println (str "this." fld " = " fld ";")))


* src/clj/cljs/compiler.clj: clean up warnings

--- src/clj/cljs/compiler.clj
+++ src/clj/cljs/compiler.clj
@@ -51,6 +51,12 @@
   [& args]
   `(.println System/err (str ~@args)))

+(defn warning [env s]
+  (binding [*out* *err*]
+    (println
+     (str s (when (:line env)
+       (str " at line " (:line env) " " *cljs-file*))))))
+
 (defn munge [s]
   (let [ss (str s)
         ms (if (.contains ss "]")
@@ -68,11 +74,8 @@
     (let [crnt-ns (-> env :ns :name)]
       (when (= prefix crnt-ns)
         (when-not (-> @namespaces crnt-ns :defs suffix)
-          (binding [*out* *err*]
-            (println
-              (str "WARNING: Use of undeclared Var " prefix "/" suffix
-                   (when (:line env)
-                     (str " at line " (:line env)))))))))))
+          (warning env
+            (str "WARNING: Use of undeclared Var " prefix "/" suffix)))))))

 (defn resolve-ns-alias [env name]
   (let [sym (symbol name)]
@@ -179,10 +182,8 @@
                  ;; dependency ordering happens *after* compilation
                  (= (:ns ev) *cljs-ns*)
                  ev (not (-> ev :dynamic)))
-        (binding [*out* *err*]
-          (println (str "WARNING: " (:name-sym ev) " not declared ^:dynamic"
-                        (when (:line env)
-                          (str " at line " (:line env) " " *cljs-file*)))))))))
+        (warning env
+          (str "WARNING: " (:name-sym ev) " not declared ^:dynamic"))))))

 (defn- comma-sep [xs]
   (interpose "," xs))
@@ -835,11 +836,9 @@
                       (get-in @namespaces [ns-name :uses sym]))
                 (let [ev (resolve-existing-var (dissoc env :locals) sym)]
                   (when *cljs-warn-on-redef*
-                    (binding [*out* *err*]
-                      (println (str "WARNING: " sym " already refers to: " (symbol (str (:ns ev)) (str sym))
-                                    " being replaced by: " (symbol (str ns-name) (str sym))
-                                    (when (:line env)
-                                      (str " at line " (:line env) " " *cljs-file*))))))
+                    (warning env
+                      (str "WARNING: " sym " already refers to: " (symbol (str (:ns ev)) (str sym))
+                           " being replaced by: " (symbol (str ns-name) (str sym)))))
                   (swap! namespaces update-in [ns-name :excludes] conj sym)
                   (update-in env [:ns :excludes] conj sym))
                 env)
@@ -855,11 +854,9 @@
         (when (and *cljs-warn-on-fn-var*
                    (not (-> sym meta :declared))
                    (and (:fn-var v) (not fn-var?)))
-          (binding [*out* *err*]
-            (println (str "WARNING: " (symbol (str ns-name) (str sym))
-                          " no longer fn, references are stale"
-                          (when (:line env)
-                            (str ", at line " (:line env) " " *cljs-file*)))))))
+          (warning env
+            (str "WARNING: " (symbol (str ns-name) (str sym))
+                 " no longer fn, references are stale"))))
       (swap! namespaces update-in [ns-name :defs sym]
              (fn [m]
                (let [m (assoc (or m {}) :name name)]


Remove unused function

Remove unused function

--- src/clj/momentum/http.clj
+++ src/clj/momentum/http.clj
@@ -6,7 +6,3 @@

 (import-fn #'response/respond)
 (import-macro #'endpoint/endpoint)
-
-(defn websocket?
-  [request]
-  (= "websocket" (request "upgrade")))


remove unnecessary logging

remove unnecessary logging

--- src/clj/tictag/server.clj
+++ src/clj/tictag/server.clj
@@ -257,7 +257,6 @@
                         {:db db :jwt jwt}
                         (:username params)
                         (:password params))]
-    (timbre/debugf "%s" params)
     (if valid?
       {:status 200 :headers {} :body token}
       {:status 401 :headers {} :body nil})))
@@ -379,7 +378,6 @@
     (response {})))

 (defn tagtime-import-from-file [{:keys [db ws]} {:keys [multipart-params user-id params] :as req}]
-  (debug multipart-params)
   (taoensso.timbre/logged-future
    (if-let [parsed-all (seq
                         (partition-all


TODOs

TODOs

--- src/clj/zetawar/data.cljs
+++ src/clj/zetawar/data.cljs
@@ -173,7 +173,6 @@
                 :armored   9}
                :zoc
                [:personnel :armored]}
-   ;; TODO: mortar should not exert zone of control
    :mortar {:name "Mortar"
             :cost 200
             :movement 9
@@ -277,6 +276,7 @@
    })

 ;; TODO: remove redundant id keys (?)
+;; TODO: rename :name to :description (?)
 (def maps
   {:sterlings-aruba
    {:id :sterlings-aruba


Create a function wrapper for a cleanly exposed API.

--- src/clj_beautify/core.clj
+++ src/clj_beautify/core.clj
@@ -2,6 +2,11 @@
   (:require [clj-beautify.file-handler :as f]
             [clj-beautify.beautify :refer [format-clj]])
   (:gen-class))
+
+(defn format-string
+  "Just a function wrapper arond format-clj"
+  [input mode]
+  (format-clj input mode))

 (defn format-file
   "Given a file page and a valid mode (`clj`|`edn`) open and use
@@ -10,7 +15,7 @@
   file with a formatted string."
   [filename mode]
   (let [input   (f/read-file filename)
-        output  (format-clj input mode)]
+        output  (format-string input mode)]
     (f/write-file filename output)))

 (defn -main


fix transit reader to deal with js objects for speed

--- src/cljs/common/util.cljs
+++ src/cljs/common/util.cljs
@@ -19,5 +19,6 @@
     (t/write w x)))

 (defn js->cljs [x]
-  (let [r (t/reader :json)]
-    (t/read r x)))
+  (let [r   (t/reader :json)
+        str (if-not (string? x) (.stringify js/JSON x) x)]
+    (t/read r str)))


More work with Top nav bar

More work with Top nav bar

--- src/cljs/metime/core.cljs
+++ src/cljs/metime/core.cljs
@@ -38,14 +38,14 @@
               [:li {:class (when (= (:active item) true) "active")} [:a {:href (:path item)} (:text item)]])])))

-(defn refresh-navigation []
-  (
-   let [token (.getToken history)
-        set-active (fn [nav]
-                     ((if (= (:path item) token) (assoc item :active true) (assoc item :active false))))
-        set-menu-items (fn [_] (update-in @app-state [:top-nav-bar] #(map set-active %)))]
-
-   (swap! app-state #(map set-active %))))
+;; (defn refresh-navigation []
+;;   (
+;;    let [token (.getToken history)
+;;         set-active (fn [nav]
+;;                      ((if (= (:path item) token) (assoc item :active true) (assoc item :active false))))
+;;         set-menu-items (fn [_] (update-in @app-state [:top-nav-bar] #(map set-active %)))]
+
+;;    (swap! app-state #(map set-active %))))

 ;; (refresh-navigation)

@@ -60,7 +60,7 @@
 (def history (History.))

 (defn on-navigate [event]
-  (refresh-navigation)
+  ;;(refresh-navigation)
   (secretary/dispatch! (.-token event)))

 (defroute "/employees/:id" [id]


started work on creating a defmapop that'll delete the temp file properly.

--- src/clj/forma/hdf.clj
+++ src/clj/forma/hdf.clj
@@ -20,18 +20,18 @@
 (def nasa-dir-path
   "/Users/sritchie/Desktop/MODISTEST/")

-(defmapop [unpack [somearg]] {:stateful true}
+;; TODO -- figure out how to delete values after we create them.
+(defmapop [unpack [somearg]] {:stateful true}
   ([]
-     (let [hash (Integer/toString (m/abs (.hashCode key)))]
-       (do (gdal/AllRegister)
-           (i/file (io/temp-dir "hdf") hash))))
+     (gdal/AllRegister))
   ([tfile stream]
-     (let [tpath (.toString tfile)
+     (let [hash (Integer/toString (m/abs (.hashCode stream)))
+           tfile (i/file (io/temp-dir "hdf") hash)
            bytes (byte-array (.getLength stream)
                              (.getBytes stream))]
        (do
          (i/copy bytes tfile)
-         (gdal/Open tpath 0))))
+         (gdal/Open (.toString tfile) 0))))
   ([tfile]
      (i/delete-file tfile)))


Trash-cost-bonus.

Trash-cost-bonus.

--- src/clj/game/core.clj
+++ src/clj/game/core.clj
@@ -400,6 +400,14 @@
       (+ (or (get-in @state [:bonus :cost]) 0))
       (max 0)))

+(defn trash-cost-bonus [state side n]
+  (swap! state update-in [:bonus :trash] (fnil #(+ % n) 0)))
+
+(defn trash-cost [state side {:keys [cost] :as card}]
+  (-> cost
+      (+ (or (get-in @state [:bonus :trash]) 0))
+      (max 0)))
+
 (defn damage-count [state side dtype n {:keys [unpreventable unboostable] :as args}]
   (-> n
       (+ (or (when (not unboostable) (get-in @state [:damage :damage-bonus dtype])) 0))
@@ -612,7 +620,7 @@
         (when-let [access-effect (:access cdef)]
           (resolve-ability state (to-keyword (:side c)) access-effect c nil))
         (when (not= (:zone c) [:discard])
-          (if-let [trash-cost (:trash c)]
+          (if-let [trash-cost (trash-cost state side (:trash c))]
             (let [card (assoc c :seen true)]
               (optional-ability state :runner card (str "Pay " trash-cost "[Credits] to trash " name "?")
                                 {:cost [:credit trash-cost]


Remove debug line

Remove debug line

--- src/cljs/ji/main.cljs
+++ src/cljs/ji/main.cljs
@@ -70,7 +70,6 @@
          (if (>= (count cs) 3)
            (do (>! c cs) (recur #{}))
            (let [v (<! card-chan)]
-             (println "selected" v)
              (cond
                (nil? v) (close! c)
                (cs v) (recur (disj cs v))


moved clustering codebase to its own routine

--- src/crawler/extractor.clj
+++ src/crawler/extractor.clj
@@ -11,7 +11,6 @@
             [crawler.rank :as rank]
             [crawler.records :as records]
             [crawler.utils :as utils]
-            [itsy.core :as itsy]
             [org.bovinegenius [exploding-fish :as uri]])
   (:use [clojure.tools.logging :only (info error)]
         [clj-logging-config.log4j])
@@ -377,9 +376,52 @@
         xpath-expls)))
     explorations)))

+(defn cluster-member?
+  [a-cluster x]
+  (some
+   (fn [a-member]
+     (>= (page/signature-edit-distance-similarity
+          (-> a-member :r1)
+          (-> x :r1)
+          0.25
+          0.75)
+         *page-sim-thresh*))
+   a-cluster))
+
 (defn cluster-explorations
+  "Explorations are converted to an exploration-ds and
+   these are clustered together. We don't assign the
+   source URL into the clusters (is this a good idea?)"
+  [{url                  :src-link
+    explorations         :explorations
+    xpaths-hrefs         :xpaths-hrefs
+    signature            :signature
+    dfs                  :dfs
+    hrefs                :hrefs
+    novelties            :novelties
+    updates              :updates
+    in-host-xpaths-hrefs :in-host-xpaths-hrefs}]
+
+  (let [explorations-ds (exploration->exploration-ds explorations)]
+
+    (reduce
+     (fn [clusters x]
+       (cluster/assign
+        (into [] clusters) x cluster-member?))
+     []
+     explorations-ds)))
+
+(defn process-page-cluster
+  "Page clusters are explored"
+  [explorations-ds-clustered]
+
+  '*)
+
+(defn cluster-explorations-titanic
   "Explorations are grouped into
-clusters based on their similarities."
+clusters based on their similarities.
+
+titanic because this guy is going down"
   [{url                  :src-link
     explorations         :explorations
     xpaths-hrefs         :xpaths-hrefs
@@ -390,32 +432,11 @@
     updates              :updates
     in-host-xpaths-hrefs :in-host-xpaths-hrefs}]

-  (let [explorations-ds (exploration->exploration-ds explorations)
-
-        cluster-member? (fn [a-cluster x]
-                          (some
-                           (fn [a-member]
-                             (>= (page/signature-edit-distance-similarity
-                                  (-> a-member :r1)
-                                  (-> x :r1)
-                                  0.25
-                                  0.75)
-                                 *page-sim-thresh*))
-                           a-cluster))
-
-        clusters        (reduce
-                         (fn [clusters x]
-                           (cluster/assign
-                            (into [] clusters) x cluster-member?))
-                         []
-                         explorations-ds)
-
-        self-cluster    (cluster/assign-where?
-                         clusters
-                         {:r1 signature
-                          :incoming-xpath nil
-                          :url url}
-                         cluster-member?)
+  (let [clusters        '*
+
+        self-cluster    '*
+
+        explorations-ds '*

         content-candids (map
                          #(nth clusters %)


basic writer added

basic writer added

--- src/crawler/crawl.clj
+++ src/crawler/crawl.clj
@@ -1,12 +1,17 @@
 (ns crawler.crawl
   "Initial crawl setup"
   (:require [clj-http.client :as client]
-            [crawler.rich-extractor :as extractor]))
+            [clojure.java.io :as io]
+            [crawler.rich-extractor :as extractor])
+  (:use [clojure.pprint :only [pprint]]))

 (defn write
-  [content]
-  (println :write (content :url)))
+  [content filename]
+  (pprint
+   content
+   (io/writer
+    filename :append true)))

 (defn distinct-by-key
  [coll k]
@@ -76,6 +81,10 @@
       (do
         (println :total-score (/ sum-score num-decisions))
         (println :cur-page-score score)
+        (write
+         {:url url
+          :body body}
+         "crawl.json")
         (if (or paginated?
                 (< (* 0.75
                       (/ sum-score num-decisions))


dry up url-ends-with assertions

dry up url-ends-with assertions

--- pedestal-app/test/semtag_web/integration_test.clj
+++ pedestal-app/test/semtag_web/integration_test.clj
@@ -28,9 +28,6 @@
   (original-report m)
   (taxi/take-screenshot :file (screenshot-file ".clojure-test-error-")))

-(defn app-url [& args]
-  (apply str "/semtag-web-test.html" args))
-
 (use-fixtures :each
               (fn [f]
                 (taxi/set-driver! (init-driver {:webdriver (PhantomJSDriver. (DesiredCapabilities. ))}))
@@ -40,9 +37,16 @@
                 (binding [ctest/report report] (f))
                 (taxi/quit)))

+;; Test Helpers
+(defn app-url [& args]
+  (apply str "/semtag-web-test.html" args))
+
 (defn click [text]
   (core/click (taxi/find-element {:tag :a :text text}))
   (Thread/sleep 500))
+
+(defn url-ends-with [& args]
+  (is (.endsWith (taxi/current-url) (apply app-url args))))

 ;; TODO - revisit not being able to go forward - log count stays the same going forward
 #_(deftest history-works
@@ -54,24 +58,25 @@

 (deftest urls-get-updated-correctly
   (is (.endsWith (taxi/current-url) (app-url)))
+  (url-ends-with "")
   (click "Tag Stats")
-  (is (.endsWith (taxi/current-url) (app-url "#/tag-stats")))
+  (url-ends-with "#/tag-stats")

   (click "Type Stats")
-  (is (.endsWith (taxi/current-url) (app-url "#/types")))
+  (url-ends-with "#/types")

   (click "All The Things")
-  (is (.endsWith (taxi/current-url) (app-url "#/all")))
+  (url-ends-with "#/all")

   (click "Home")
-  (is (.endsWith (taxi/current-url) (app-url "#/"))))
+  (url-ends-with "#/"))

 (deftest search-submit-works
   (taxi/input-text "#url_search_text" "feynman")
-  (is (= "feynman" (taxi/attribute "#url_search_text" :value)))
-
   (taxi/click "#url_search_button")
   (Thread/sleep 1000)
+
+  (url-ends-with "#/search?query=feynman&search-type=tagged")
   (is (= "Search results for 'feynman'" (taxi/text "#search_title"))))

 (comment


FIX #18

FIX #18

--- resources/leiningen/new/closp/clj/routes/user.clj
+++ resources/leiningen/new/closp/clj/routes/user.clj
@@ -62,8 +62,9 @@
                  {:account_activated_title (t locale tconfig :user/account_activated_title)
                   :account_activated       (t locale tconfig :user/account_activated)}))

-(defn signup-page [{:keys [captcha-public-key]} & [errormap]]
-  (layout/render "user/signup.html" (merge {:captcha-public-key captcha-public-key} errormap)))
+(defn signup-page [{:keys [captcha-public-key captcha-enabled?]} & [errormap]]
+  (layout/render "user/signup.html" (merge {:captcha-public-key captcha-public-key
+                                            :captcha-enabled? captcha-enabled?} errormap)))

 (defn activate-account [config id locale tconfig]
   (if (db/get-user-by-act-id id)


Fix indentation

Fix indentation

--- dev-resources/user.clj
+++ dev-resources/user.clj
@@ -28,11 +28,11 @@
    (make-event Level/ERROR msg fields))
   ([level msg fields]
    (doto (LoggingEvent. "fully-qualified-class-name"
-                                                   logger
-                                                   level
-                                                   msg
-                                                   nil
-                                                   nil)
+                        logger
+                        level
+                        msg
+                        nil
+                        nil)
      (.setTimeStamp (System/currentTimeMillis))
      (.setMarker (make-marker))
      (.setMDCPropertyMap fields))))