Purdyup scheduling a bit

Purdyup scheduling a bit

--- src/demo/scheduling.clj
+++ src/demo/scheduling.clj
@@ -1,5 +1,5 @@
 (ns demo.scheduling
-  (:require [immutant.scheduling :as sch]
+  (:require [immutant.scheduling :refer (every at in until limit cron) :as sch]
             [immutant.util       :as util]
             immutant.scheduling.joda
             clj-time.core
@@ -9,32 +9,32 @@
 ;; scheduling specs are just maps. we provide optional helpers for
 ;; generating thoes maps
 (def every-5s
-  (or {:every (or 5000
-                [5 :seconds])}
-    (sch/every 5 :seconds)))
+  (or
+    {:every (or 5000 [5 :seconds])}
+    (every 5 :seconds)))

 ;; :at can be a Date, a time as a string, or millis since epoch
 (def daily
-  (-> (sch/at (or (Date.) "1830" 1395439646983))
-    (sch/every :day)))
+  (-> (at (or (Date.) "1830" 1395439646983))
+    (every :day)))

 ;; helpers optionally take a map as the first arg, so compose
 (def in-5m-until-5pm
-  (-> (sch/in 5 :minutes)
-    (sch/every 2 :hours, 30 :minutes)
-    (sch/until "1700")))
+  (-> (in 5 :minutes)
+    (every 2 :hours, 30 :minutes)
+    (until "1700")))

 ;; cron-style strings are also supported
 (def nine-am
-  (sch/cron "0 0 9 * * ?"))
+  (cron "0 0 9 * * ?"))

 (def every-5s-cron
-  (sch/cron "*/5 * * * * ?"))
+  (cron "*/5 * * * * ?"))

 (def every-10ms-in-500ms-4-times
-  (-> (sch/in 500)
-    (sch/every 10)
-    (sch/limit 4)))
+  (-> (in 500)
+    (every 10)
+    (limit 4)))

 ;; you can use clj-time periods if using clj-time
 (defn every-3s-lazy-seq []
@@ -48,7 +48,7 @@
   (def every-2-job
     (sch/schedule (bound-fn []
                     (println "called"))
-      (sch/every 2 :seconds)))
+      (every 2 :seconds)))

   ;; the return value of the schedule call can be used to stop it
   (sch/stop every-2-job)
@@ -73,16 +73,16 @@
         (println "unscheduling beep & boop")
         (sch/stop beep)
         (sch/stop boop))
-      (sch/in 20 :seconds)))
+      (in 20 :seconds)))

   ;; start singleton and non-singleton jobs to demo cluster failover
   (when (util/in-container?)

     ;; singleton jobs require an id
     (sch/schedule #(println "I run on ONE node")
-      (-> (sch/every 10 :seconds)
-        (sch/id :a_singleton)))
+      (-> {:id :a-unique-id-for-the-singleton}
+        (every 10 :seconds)))

     (sch/schedule #(println "I run on EVERY node")
-      (-> (sch/every 9 :seconds)
-        (sch/singleton false)))))
+      (-> {:singleton false}
+        (every 9 :seconds)))))


Add an override for the return coordinate vector


fail to map Alaska. but it was fun making an initial attempt

--- src/r_incantations/core.clj
+++ src/r_incantations/core.clj
@@ -250,6 +250,13 @@
       (incanter.charts/add-function incanter.stats/pdf-gamma 0 8)
       (incanter.core/view))

+  ;; fails to map alaska nicely
+  (let [ak (incanter.core/$where {:region "USA" :subregion "Alaska"} world)
+        coords (incanter.core/sel ak :cols [:long :lat])
+        coords (incanter.core/to-matrix coords)
+        p (incanter.charts/scatter-plot (map first coords) (map second coords))
+        p (incanter.charts/add-polygon p coords)]
+    (incanter.core/view p))

   )


comment typo

comment typo

--- src/re_frame/registrar.cljc
+++ src/re_frame/registrar.cljc
@@ -1,6 +1,6 @@
 (ns re-frame.registrar
   "In many places, re-frame asks you to associate an `id` (keyword)
-  with a `handler` (fucntion).  This namespace contains the
+  with a `handler` (function).  This namespace contains the
   central registry of such associations."
   (:require  [re-frame.interop :refer [debug-enabled?]]
              [re-frame.loggers    :refer [console]]))


Minor fix: exception should be wrapped by -main

--- src/photon/core.clj
+++ src/photon/core.clj
@@ -74,16 +74,16 @@
 ;; In order to use http-kit, run `lein run` instead of `lein ring server`
 (defn photon-component [conf]
   (log/info "Starting photon...")
+  (let [system (photon-system conf)
+        comp {:web-server (component/using (web-server conf) [:ui])}
+        web-system (merge system comp)]
+    (component/start web-system)))
+
+(defn -main [& args]
   (try
-    (let [system (photon-system conf)
-          comp {:web-server (component/using (web-server conf) [:ui])}
-          web-system (merge system comp)]
-      (component/start web-system))
+    (photon-component (apply conf/config args))
     (catch UnsupportedOperationException e
       (println (.getMessage e)))))
-
-(defn -main [& args]
-  (photon-component (apply conf/config args)))

 (defonce figwheel-instance (ref nil))


Remove println

Remove println

--- src/grafter/tabular.clj
+++ src/grafter/tabular.clj
@@ -218,7 +218,6 @@
   (let [args-from-cols (select-row-values src-col-ids row)
         new-col-val (apply f args-from-cols)
         new-column-hash (resolve-keys new-header new-col-val)]
-    (println new-column-hash)
     (merge row new-column-hash)))

 (defn- resolve-all-col-ids [dataset source-cols]


added sever-logfile

added sever-logfile

--- sidecar/src/figwheel_sidecar/config_check/validate_config.clj
+++ sidecar/src/figwheel_sidecar/config_check/validate_config.clj
@@ -173,6 +173,7 @@
            :css-dirs          [string?]
            :ring-handler      (ref-schema 'Named)
            :reload-clj-files  (ref-schema 'ReloadCljFiles)
+           :server-logfile    string?
            :open-file-command string?
            :repl              (ref-schema 'Boolean)
            :nrepl-port        integer?
@@ -243,9 +244,6 @@
           :cljsbuild (ref-schema 'CljsbuildOptions)})
    figwheel-cljsbuild-rules))

-
-
-
 ;; alright everyting below is the result of being so wishywashy about
 ;; configuration and now I'm having to support multiple ways of doing things


adding missing clojurescript compiler config keys

--- sidecar/src/figwheel_sidecar/config_check/validate_config.clj
+++ sidecar/src/figwheel_sidecar/config_check/validate_config.clj
@@ -13,14 +13,6 @@
    [fipp.engine :refer [pprint-document]]
    [clojure.string :as string]
    [clojure.java.io :as io]))
-
-(defn fmt-doc
-  ([s] s)
-  ([s ex]
-   [:group (fmt-doc s)
-    :line
-    :line
-    [:nest 2 ex] :line]))

 (def CompilerWarningsSpec
   (->>
@@ -92,12 +84,7 @@
           :verbose                   (ref-schema 'Boolean)
           :pretty-print              (ref-schema 'Boolean)
           :target                    :nodejs
-          :foreign-libs              [{:file string?
-                                       :provides [string?]
-                                       :file-min string?
-                                       :requires [string?]
-                                       :module-type (ref-schema 'ModuleType)
-                                       :preprocess (ref-schema 'Named)}]
+          :foreign-libs              [(ref-schema 'ForeignLib)]
           :externs                   [string?]
                                      ;; TODO keys only work with direct predicates
           :modules                   {named?
@@ -125,7 +112,22 @@
           :anon-fn-naming-policy     (ref-schema 'AnonFnNamingPolicy)
           :optimize-constants        (ref-schema 'Boolean)
           :parallel-build            (ref-schema 'Boolean)
-          :devcards                  (ref-schema 'Boolean)})
+          :devcards                  (ref-schema 'Boolean)
+
+          :dump-core                 (ref-schema 'Boolean)
+          :emit-constants            (ref-schema 'Boolean)
+
+          :warning-handlers          [anything?]
+          :source-map-inline         (ref-schema 'Boolean)
+          :ups-libs                  [string?]
+          :ups-externs               [string?]
+          :ups-foreign-libs          [(ref-schema 'ForeignLib)]})
+    (spec 'ForeignLib {:file string?
+                       :provides [string?]
+                       :file-min string?
+                       :requires [string?]
+                       :module-type (ref-schema 'ModuleType)
+                       :preprocess (ref-schema 'Named)} )
    (or-spec 'CompilerWarnings
             (ref-schema 'Boolean)
             CompilerWarningsSpec)
@@ -241,7 +243,8 @@
           :cljsbuild (ref-schema 'CljsbuildOptions)})
    figwheel-cljsbuild-rules))

-(def schema-rules (index-spec schema-rules-base))
+
+

 ;; alright everyting below is the result of being so wishywashy about
 ;; configuration and now I'm having to support multiple ways of doing things
@@ -307,10 +310,17 @@
                                                          [cljb-k cljb-v]]))]
         (print-one-error (validate-regular-rules conf) 'RootMap conf)))))

-#_(validate-project-config {:cljsbuild
-                            {:builds [{:id "dev"
-                                       :source-path 1
-                                       :compiler {}}] }})
+#_ (with-color
+     (validate-project-config
+   {:cljsbuild
+    {:builds { :source-paths ["src" ]
+              :fighweel true
+              :compiler {}}
+
+                                        }
+    :figwheel {}
+                            }
+                           ))

 #_(defn tester [conf]
     (with-schema (validate-regular-rules {:figwheel {}})
@@ -400,12 +410,14 @@

 (comment
   ;; figure out
+  (def schema-rulest (index-spec schema-rules-base))
+
   (defn get-all-keys [rules]
     (for [[k & ks] (rules :-)] k))

   (distinct
-   (for [k (get-all-keys schema-rules)
-        k2 (get-all-keys schema-rules)
+   (for [k (get-all-keys schema-rulest)
+        k2 (get-all-keys schema-rulest)
          :when (and
                 (not= k k2)
                 (tc/similar-key 0 k k2))]


Fix error in build check script.

Fix error in build check script.

git-svn-id: 03a8048b5ee8463be5048a3801110fb50f378627@11570 d0ab736e-dc22-4aeb-8dc9-08def0aa14fd

--- buildscripts/buildcheck/src/buildcheck/core.clj
+++ buildscripts/buildcheck/src/buildcheck/core.clj
@@ -155,13 +155,14 @@
   (let [installer32-ok (exe-on-server? 32 today-token)
         installer64-ok (exe-on-server? 64 today-token)
         mac-ok (mac-build-on-server? today-token)
-        missing-vcproj-files (missing-vcproj)]
+        missing-vcproj-files (missing-vcproj)
+        missing-links (missing-device-links)]
     (when-not (and (not testmode)
                    installer32-ok
                    installer64-ok
                    mac-ok
                    (empty? missing-vcproj-files)
-                   (empty? missing-device-links))
+                   (empty? missing-links))
       (str
         "MICROMANAGER BUILD STATUS REPORT\n"
         "\n\nIs Windows 32-bit installer download available on website?\n"
@@ -173,7 +174,7 @@
         (report-segment "Missing .vcproj files" missing-vcproj-files)
         (report-segment "Uncompiled device adapters (Win32)" (missing-device-adapters 32))
         (report-segment "Uncompiled device adapters (x64)" (missing-device-adapters 64))
-        (report-segment "Missing device links" (missing-device-links))
+        (report-segment "Missing device links" (missing-links))
         (report-segment "Missing device pages" (missing-device-pages))))))

 (defn make-full-report [send?]


Start to simplify todomvc

Start to simplify todomvc

--- examples/todomvc/src/todomvc/handlers.cljs
+++ examples/todomvc/src/todomvc/handlers.cljs
@@ -10,12 +10,13 @@
 ;; -- Middleware --------------------------------------------------------------
 ;;
 ;; checks that the structure in app-db matches the schema
-(def check-schema (after valid-schema?))
+(def schema-ok? (after valid-schema?))

+;; store todos in local storage
 (def write-ls-todos (after #(set-local-storage! :todos %)))

-;; middleware for any handler which manipulates todos.
-(def todo-middleware [check-schema (path [:todos]) write-ls-todos debug  trim-v])
+;; middleware for a handler which manipulates todos.
+(def todo-middleware [schema-ok? (path :todos) write-ls-todos debug  trim-v])

 ;; -- Helpers -----------------------------------------------------------------
@@ -39,7 +40,7 @@

 (register-handler                 ;; handlers changes the footer filter
   :set-showing                    ;; event-id
-  [(path [:showing]) #_write-ls check-schema debug trim-v]     ;; middleware  (wraps the handler)
+  [(path :showing) check-schema debug trim-v]     ;; middleware  (wraps the handler)
   (fn                             ;; handler
     [db [filter-kw]]
     filter-kw))


Move flush! into ITable protocol, switch to basic extend pattern.

--- src/merkle_db/table.clj
+++ src/merkle_db/table.clj
@@ -127,6 +127,15 @@
     "Remove some records from the table, identified by a collection of id keys.
     Returns an updated table.")

+  (flush!
+    [table]
+    [table apply-patches?]
+    "Ensure that all local state has been persisted to the storage backend and
+    return an updated persisted table.
+
+    If `apply-patches?` is true, the current patch data will be merged into the
+    main data tree and the returned table will have no patch link.")
+
   ;; Partitions

   (list-partitions
@@ -383,22 +392,21 @@
   (.dirty table))

-; TODO: put this in protocol?
-(defn flush!
+(defn- -flush!
   "Ensure that all local state has been persisted to the storage backend and
   return an updated non-dirty table."
   ([table]
-   (flush! table false))
+   (-flush! table false))
   ([^Table table apply-patches?]
    (if (dirty? table)
      (let [[patch-link data-link]
              (if (or (seq (.pending table)) (::patch table))
                (if apply-patches?
                  ; Combine pending changes and patch tablet and update data tree.
-                 [nil (throw (UnsupportedOperationException. "updated data-tree link"))]
+                 [nil (throw (UnsupportedOperationException. "NYI: merge pending changes and patch tablet into data tree"))]
                  ; flush any pending changes to the patch tablet
                  ; if patch tablet overflows, update main data tree
-                 (throw (UnsupportedOperationException. "NYI")))
+                 (throw (UnsupportedOperationException. "NYI: flush pending changes to patch tablet, overflow to data tree")))
                ; No patch data or tablet.
                [nil (::data table)])
            root-data (-> (.root-data table)
@@ -408,20 +416,22 @@
                            patch-link (assoc ::patch patch-link)
                            data-link (assoc ::data data-link)))]
        (when-not (s/valid? ::node-data root-data)
-         (throw (ex-info
-                  "Cannot serialize invalid root data"
-                  {:type ::invalid-root
-                   :errors (s/explain-data ::node-data root-data)})))
-       (let [node (mdag/store-node! (.store table) nil root-data)]
+         (throw (ex-info (str "Cannot write invalid table root node: "
+                              (s/explain-str ::node-data root-data))
+                         {:type ::invalid-root})))
+       (let [node (mdag/store-node! (.store table) nil root-data)
+             table-info (assoc (.table-info table)
+                               ::node/id (::node/id node)
+                               ::data/size (node/reachable-size node))
+             table-meta (assoc (._meta table)
+                               ::node/links (::node/links node))]
          (->Table
            (.store table)
-           (assoc (.table-info table)
-                  ::node/id (::node/id node)
-                  ::data/size (node/reachable-size node))
+           table-info
            (::node/data node)
-           nil
+           (sorted-map-by key/compare)
            false
-           (._meta table))))
+           table-meta)))
      ; Table is clean, return directly.
      table)))

@@ -443,92 +453,98 @@

 (defn- -scan
-  [^Table table opts]
-  (let [lexicoder (table-lexicoder table)
-        start-key (some->> (:start-key opts) (key/encode lexicoder))
-        end-key (some->> (:end-key opts) (key/encode lexicoder))]
-    (->>
-      (patch/patch-seq
-        ; Merged patch data to apply to the records.
-        (patch/filter-patch
-          (.pending table) ; TODO: merge ::patch
-          {:fields (:fields opts)
-           :start-key start-key
-           :end-key end-key})
-        ; Lazy sequence of matching records from the index tree.
-        (when-let [data-node (mdag/get-data
-                               (.store table)
-                               (::data (.root-data table)))]
-          (if (or start-key end-key)
-            (index/read-range
-              (.store table)
-              data-node
-              (:fields opts)
-              start-key
-              end-key)
-            (index/read-all
-              (.store table)
-              data-node
-              (:fields opts)))))
-      (patch/remove-tombstones)
-      (map (key-decoder lexicoder)))))
+  ([table]
+   (-scan table nil))
+  ([^Table table opts]
+   (let [lexicoder (table-lexicoder table)
+         start-key (some->> (:start-key opts) (key/encode lexicoder))
+         end-key (some->> (:end-key opts) (key/encode lexicoder))]
+     (->>
+       (patch/patch-seq
+         ; Merged patch data to apply to the records.
+         (patch/filter-patch
+           (.pending table) ; TODO: merge ::patch
+           {:fields (:fields opts)
+            :start-key start-key
+            :end-key end-key})
+         ; Lazy sequence of matching records from the index tree.
+         (when-let [data-node (mdag/get-data
+                                (.store table)
+                                (::data (.root-data table)))]
+           (if (or start-key end-key)
+             (index/read-range
+               (.store table)
+               data-node
+               (:fields opts)
+               start-key
+               end-key)
+             (index/read-all
+               (.store table)
+               data-node
+               (:fields opts)))))
+       (patch/remove-tombstones)
+       (map (key-decoder lexicoder))))))

 (defn- -get-records
-  [^Table table id-keys opts]
-  (let [lexicoder (table-lexicoder table)
-        id-keys (into #{} (map (partial key/encode lexicoder)) id-keys)
-        patch-map (.pending table) ; TODO: merge ::patch
-        patch-entries (select-keys patch-map id-keys)
-        extra-keys (apply disj id-keys (keys patch-entries))
-        patch-entries (patch/filter-patch patch-entries {:fields (:fields opts)})
-        data-entries (when-let [data-node (and (seq extra-keys)
-                                               (mdag/get-data
-                                                 (.store table)
-                                                 (::data (.root-data table))))]
-                       (index/read-batch
-                         (.store table)
-                         data-node
-                         (:fields opts)
-                         extra-keys))]
-    (->> (concat patch-entries data-entries)
-         (patch/remove-tombstones)
-         (map (key-decoder lexicoder)))))
+  ([table id-keys]
+   (-get-records table id-keys nil))
+  ([^Table table id-keys opts]
+   (let [lexicoder (table-lexicoder table)
+         id-keys (into #{} (map (partial key/encode lexicoder)) id-keys)
+         patch-map (.pending table) ; TODO: merge ::patch
+         patch-entries (select-keys patch-map id-keys)
+         extra-keys (apply disj id-keys (keys patch-entries))
+         patch-entries (patch/filter-patch patch-entries {:fields (:fields opts)})
+         data-entries (when-let [data-node (and (seq extra-keys)
+                                                (mdag/get-data
+                                                  (.store table)
+                                                  (::data (.root-data table))))]
+                        (index/read-batch
+                          (.store table)
+                          data-node
+                          (:fields opts)
+                          extra-keys))]
+     (->> (concat patch-entries data-entries)
+          (patch/remove-tombstones)
+          (map (key-decoder lexicoder))))))

 (defn- -insert
-  [table records opts]
-  (let [{:keys [merge-record merge-field]} opts
-        lexicoder (table-lexicoder table)
-        update-record (cond
-                        merge-record merge-record
-                        merge-field
-                        (fn merge-fields
-                          [_ old-data new-data]
-                          (reduce
-                            (fn [data field-key]
-                              (assoc data
-                                     field-key
-                                     (merge-field field-key
-                                                  (get old-data field-key)
-                                                  (get new-data field-key))))
-                            {} (distinct (concat (keys old-data) (keys new-data)))))
-                        :else
-                        (fn merge-simple
-                          [_ old-data new-data]
-                          (into {}
-                                (filter (comp some? val))
-                                (merge old-data new-data))))
-        extant (into {} (get-records table (map first records)))
-        new-records (map
-                      (fn [[k data]]
-                        [(key/encode lexicoder k)
-                         (update-record k (get extant k) data)])
-                      records)]
-    ; Add new data maps to pending changes.
-    (-> table
-        (update-pending into new-records)
-        (update ::data/count + (- (count records) (count extant))))))
+  ([table records]
+   (-insert table records nil))
+  ([table records opts]
+   (let [{:keys [merge-record merge-field]} opts
+         lexicoder (table-lexicoder table)
+         update-record (cond
+                         merge-record merge-record
+                         merge-field
+                         (fn merge-fields
+                           [_ old-data new-data]
+                           (reduce
+                             (fn [data field-key]
+                               (assoc data
+                                      field-key
+                                      (merge-field field-key
+                                                   (get old-data field-key)
+                                                   (get new-data field-key))))
+                             {} (distinct (concat (keys old-data) (keys new-data)))))
+                         :else
+                         (fn merge-simple
+                           [_ old-data new-data]
+                           (into {}
+                                 (filter (comp some? val))
+                                 (merge old-data new-data))))
+         extant (into {} (get-records table (map first records)))
+         new-records (map
+                       (fn [[k data]]
+                         [(key/encode lexicoder k)
+                          (update-record k (get extant k) data)])
+                       records)]
+     ; Add new data maps to pending changes.
+     (-> table
+         (update-pending into new-records)
+         (update ::data/count + (- (count records) (count extant)))))))

 (defn- -delete
@@ -545,33 +561,12 @@
         (update ::data/count - (count extant)))))

-(extend-type Table
+(extend Table

   ITable

-  ;; Records
-
-  (scan
-    ([this]
-     (-scan this nil))
-    ([this opts]
-     (-scan this opts)))
-
-
-  (get-records
-    ([this id-keys]
-     (-get-records this id-keys nil))
-    ([this id-keys opts]
-     (-get-records this id-keys opts)))
-
-
-  (insert
-    ([this records]
-     (-insert this records nil))
-    ([this records opts]
-     (-insert this records opts)))
-
-
-  (delete
-    [this id-keys]
-    (-delete this id-keys)))
+  {:scan -scan
+   :get-records -get-records
+   :insert -insert
+   :delete -delete
+   :flush! -flush!})


simplified load-dep function from pomegranate


Fix warning

Fix warning

--- src-cljs/qcon/main.cljs
+++ src-cljs/qcon/main.cljs
@@ -67,7 +67,7 @@

 (defn timeout-slide [data owner opts]
   (reify
-    om/IInitSlide
+    om/IInitState
     (init-state [_]
       {:default-font (:font-size opts)
        :status "READY"})


Update transformation.clj

Update transformation.clj

--- backend/specs/akvo/lumen/specs/transformation.clj
+++ backend/specs/akvo/lumen/specs/transformation.clj
@@ -463,7 +463,9 @@
 (s/def ::db.dsv.transformation/changedColumns* (s/keys :req-un [::db.dsv.transformation/changedColumns]))

 (s/def ::db.dsv.s/transformation (s/merge ::transformation.engine.s/op-spec ::db.dsv.transformation/changedColumns*))
-(lumen.s/sample ::transformation.engine.s/op-spec)
+
+(s/def ::op-spec ::transformation.engine.s/op-spec)
+
 (s/def ::db.dsv.s/transformations (s/coll-of ::db.dsv.s/transformation :kind vector? :distinct true))

 (s/def ::next-dataset-version (s/keys :req-un [::db.dsv.s/id ::db.dsv.s/dataset-id


[#1077] Cleanup maps

[#1077] Cleanup maps

Still WIP

--- backend/src/akvo/lumen/lib/visualisation/maps.clj
+++ backend/src/akvo/lumen/lib/visualisation/maps.clj
@@ -5,6 +5,7 @@
             [clj-http.client :as client]
             [clojure.pprint :refer [pprint]]
             [clojure.string :as s]
+            [clojure.string :as str]
             [honeysql.core :as sql]
             [honeysql.helpers :refer [values]]
             [hugsql.core :as hugsql]))
@@ -28,94 +29,73 @@
     {:headers (headers db-uri)
      :db-name db-name}))

-;;;  Dummy sql start
-(def locations
-  '({:label "HQ"
-     :longitude 4.908075
-     :latitude 52.372189}
-    {:label "Bat cave"
-     :longitude 12.069034
-     :latitude 57.74119}))
+(defn- no-dataset-sql
+  "SQL to use when there is no dataset in the visualisation spec."
+  []
+  "SELECT * FROM (VALUES ('label', ST_SetSRID(ST_MakePoint(4.908075,52.372189), 4326))) AS t (label, geom) LIMIT 0;")

-(defn sql-values [locations]
-  (s/join ", "
-          (map (fn [{:keys [label longitude latitude]}]
-                 (format "('%s', ST_SetSRID(ST_MakePoint(%s,%s), 4326))"
-                         label longitude latitude))
-               locations)))
-
-(defn sql-statement
-  "Dummy data"
-  ([]
-   "SELECT * FROM (VALUES ('label', ST_SetSRID(ST_MakePoint(4.908075,52.372189), 4326))) AS t (label, geom) LIMIT 0;")
-  ([locations]
-   (format "SELECT * FROM (VALUES %s ) AS t (label, geom);"
-           (sql-values locations))))
-
-
-(defn build-sql [table]
-  (format "SELECT c2 as label, d1 as geom FROM %s;" table))
-
-;;;  Dummy sql stop
+(defn- compile-sql
+  "Takes table geom-column & label-column strings and returns a SQL string."
+  [table geom-column label-column]
+  (let [[prepared-statement & args] (sql/format {:select [[geom-column "geom"]
+                                                          [label-column "label"]]
+                                                 :from [table]})
+        fmt-string (str/replace prepared-statement #"\?" "%s")]
+    (apply (partial format fmt-string) args)))

 (def cartocss
   "#s { marker-width: 10; marker-fill: #e00050;}")

-(defn map-config [cartocss geom-column label sql]
+(defn map-config
+  [cartocss geom-column label-column sql]
   {"version" "1.6.0"
    "layers" [{"type" "mapnik"
               "options" {"cartocss" cartocss
                          "cartocss_version" "2.0.0"
                          "geom_column" geom-column
-                         "interactivity" label
+                         "interactivity" [label-column]
                          "sql" sql
                          "srid" 4326}}]})

+(defn create
+  "Takes a tenant connection and a visualisation spec. Will get a layergroupid
+  from Windshaft and compose a response with the db-name attached under the key
+  tenantDB."
+  [tenant-conn visualisation-spec]
+  (prn "\n\n-------------------------------------------------------------------")
+  (prn "@maps/create")
+  (clojure.pprint/pprint visualisation-spec)
+  (try
+    (let [{:keys [db-name headers]} (connection-details tenant-conn)
+          akvo-maps-url "http://windshaft:4000" ;; localhost in production
+          url (format "%s/%s/layergroup" akvo-maps-url db-name)
+          dataset-id (get visualisation-spec "datasetId")
+          geom-column "d1"
+          label-column "c2"
+          sql (if (nil? dataset-id)
+                (no-dataset-sql)
+                (let [dataset (dataset-by-id tenant-conn {:id dataset-id})
+                      table (:table-name dataset)]
+                  (compile-sql table geom-column label-column)))
+          _ (prn sql)
+          map-config (map-config cartocss "geom" "label" sql)
+          windshaft-resp (client/post url
+                                      {:body (json/encode map-config)
+                                       :headers headers
+                                       :content-type :json})]
+      (lib/ok (assoc (json/decode (:body windshaft-resp))
+                     "tenantDB" db-name)))
+    (catch Exception e
+      (prn (ex-data e)))))

-(defn create [tenant-conn visualisation-spec]
-  (clojure.pprint/pprint visualisation-spec)
-  ;; Handle dataset if available, yank out relevant ds_<...>,
-  ;; geom
-  ;; interactivity
-  (let [{:keys [db-name headers]} (connection-details tenant-conn)
-        akvo-maps-url "http://windshaft:4000" ;; localhost in production
-        url (format "%s/%s/layergroup" akvo-maps-url db-name)
-        dataset-id (get visualisation-spec "datasetId")
-        sql (if (nil? dataset-id)
-              (sql-statement)
-              (let [dataset (dataset-by-id tenant-conn {:id dataset-id})
-                    table (:table-name dataset)]
-                (build-sql table)))
-        _ (prn sql)
-        resp (client/post url
-                          {:body (json/encode (map-config cartocss
-                                                          "geom"
-                                                          "label"
-                                                          sql))
-                           :headers headers
-                           :content-type :json})]
-    (case (:status resp)
-      200 (lib/ok (assoc (json/decode (:body resp))
-                         "tenantDB" db-name))
-      (prn resp))))
+

 (comment

-  ;; Playing with Honeysql as a form of AST
   (sql/format {:select [:a :b :c]
                :from [:foo]
                :where [:= :f.a "baz"]})
-
-  ;; "SELECT * FROM (VALUES %s ) AS t (label, geom);"
-  (sql/format {:select [:*]
-               :from (values [["a" 1] ["b" 2]])
-               })
-
-
-
-
-  ;; --

   ;; Example map spec
   {"type" "visualisation",
@@ -143,5 +123,4 @@
       "datasetId" "59c3ab16-34e8-44df-957e-6c1c38fbf465",
       "title" "Untitled Layer 1"}]}}

-
   )


Random shmin and Isip

Random shmin and Isip

--- src/test_utils/sandbox.cljs
+++ src/test_utils/sandbox.cljs
@@ -74,6 +74,12 @@

 (defn rand-stage-number []
   (inc (rem (tuc/rand-2) 50)))
+
+(defn rand-isip []
+  (Math/abs (tuc/draw-normal 5061 121)))
+
+(defn rand-shmin []
+  (Math/abs (tuc/draw-normal 2.29 0.05)))

 (defn rand-cluster-count []
   (inc (rand-nth (range 2 7))))


Include sliders for width and height

Include sliders for width and height

--- maps/src/maps/core.cljs
+++ maps/src/maps/core.cljs
@@ -5,15 +5,19 @@

 (defonce points (reagent/atom []))

-(defonce ticks (atom 0))
+(defonce ticks (reagent/atom 0))

-(defonce x-min (atom 0))
+(defonce x-min (reagent/atom 0))

-(defonce x-max (atom 150))
+(defonce x-max (reagent/atom 150))

-(defonce y-min (atom 0))
+(defonce y-min (reagent/atom 0))

-(defonce y-max (atom 1))
+(defonce y-max (reagent/atom 1))
+
+(defonce width (reagent/atom 800))
+
+(defonce height (reagent/atom 600))

 (def options
   {:chart       {:type "line"}
@@ -50,19 +54,19 @@
       (assoc-in [:yAxis :max] @y-max)
       (assoc-in [:series 0 :data] @points)))

-(defn chart-render [ps]
+(defn chart-render [wd ht ps]
   @ps
   [:div {:style {:min-width "310px"
-                 :max-width "800px"
-                 :height    "600px"
+                 :max-width (str @wd "px")
+                 :height    (str @ht "px")
                  :margin    "0 auto"}}])

 (defn series-updated [this]
   (js/Highcharts.Chart. (reagent/dom-node this)
                         (clj->js (chart-config))))

-(defn chart [ps]
-  (reagent/create-class {:reagent-render       #(chart-render ps)
+(defn chart [wd ht ps]
+  (reagent/create-class {:reagent-render       #(chart-render wd ht ps)
                          :component-did-update #(series-updated %1)}))

 (defn slider [id v min max]
@@ -100,8 +104,14 @@
    [:div
     [:label {:for "y-max"} "y-axis max value"]
     [slider "y-max" y-max 1 500]]
+   [:div
+    [:label {:for "width"} "width"]
+    [slider "width" width 400 1920]]
+   [:div
+    [:label {:for "height"} "height"]
+    [slider "y-max" height 400 1080]]
    [evolution! 0.851 ticks points
-    [chart points]]])
+    [chart width height points]]])

 (reagent/render-component [page]
                           (. js/document (getElementById "app")))


Small fix, but REPL is not done yet.

Small fix, but REPL is not done yet.

--- demos/src/main/webapp/WEB-INF/classes/demos.clj
+++ demos/src/main/webapp/WEB-INF/classes/demos.clj
@@ -154,7 +154,7 @@
      "  req = new XMLHttpRequest();"
      "  req.onreadystatechange = replCallback;"
      (str "  req.open('GET', '" (url "/ajaxrepl-out") "', true);")
-     "  req.send();"
+     "  req.send(null);"
      "} "

      "function appendContent(txt) {"
@@ -200,6 +200,38 @@

 (publish ajaxrepl "/ajaxrepl")

+;;; A REPL using Refs
+
+(defn repl-session []
+  (session-get "repl-messages" (fn [] (ref []))))
+
+
+
+;; Return new content since last update
+;; if there is no new content, wait for some
+(defn ajaxrepl-out []
+  (let [repl-messages (repl-session)]
+    (dbg repl-messages)
+    (dosync
+     (let [output (reduce str
+                          (interleave @repl-messages (repeat "\n")))]
+       (dbg "sending: " output)
+       (webjure/send-output "text/plain" output))
+     (ref-set repl-messages []))))
+(webjure/publish ajaxrepl-out "/ajaxrepl-out")
+
+(defn ajaxrepl-eval [str]
+  (eval (read (new java.io.PushbackReader (new java.io.StringReader str)))))
+
+(defn ajaxrepl-in []
+  (dosync
+   (let [repl-messages (repl-session)
+         input (webjure/slurp-post-data)]
+     (alter repl-messages conj (ajaxrepl-eval input))
+     (webjure/send-output "text/plain" "OK"))))
+(webjure/publish ajaxrepl-in "/ajaxrepl-in")
+
+
 ;;; Session test

 (defh "/session" [] {:output :html}
@@ -214,32 +246,3 @@
         greeting))))

-;;; A REPL using Refs
-
-(defn repl-session []
-  (session-get "repl-messages" (fn [] (ref []))))
-
-
-
-;; Return new content since last update
-;; if there is no new content, wait for some
-(defn ajaxrepl-out []
-  (dosync
-   (let [repl-messages (repl-session)
-         msgs @(repl-session)]
-     (webjure/send-output "text/plain"
-                          (reduce str
-                                  (interleave @msgs (repeat "\n"))))
-     (ref-set repl-messages []))))
-(webjure/publish ajaxrepl-out "/ajaxrepl-out")
-
-(defn ajaxrepl-in []
-  (dosync
-   (let [repl-messages (repl-session)
-         input (webjure/slurp-post-data)]
-     (alter repl-messages conj (eval input))
-     (webjure/send-output "text/plain" "OK"))))
-(webjure/publish ajaxrepl-in "/ajaxrepl-in")
-
-
-


added datomic queries

added datomic queries

--- booking-service/src/booking_service/service.clj
+++ booking-service/src/booking_service/service.clj
@@ -20,9 +20,7 @@
             [ring.middleware.session.cookie :as cookie]
             [clojure.data.codec.base64 :as base64]

-            [booking-service.db :as db]
-
-            ))
+            [booking-service.db :as db]))

 (defn- byte-transform
@@ -213,16 +211,17 @@
   (let [{{nm :name
           tel  :tel
           selected :selected} :edn-params} req
-        customer (db/store-customer nm tel)
+
+          seat-ids (map #(let[[x y] %] (keyword (str x "_" y))) selected)
+          customer-id (db/store-customer nm tel)
+
+          booking-id (db/store-booking customer-id seat-ids)
         ]
-
-
-    (println "")
-    (println
-      (pr-str customer ))
-    (println "")
-
-    (bootstrap/edn-response [:xyj])
+
+    (bootstrap/edn-response {:status :ok
+                             :customer-id customer-id
+                             :booking-id booking-id
+                             })
     ))


Add test for POST request (JSON)

Add test for POST request (JSON)

--- planck-cljs/test/planck/http_test.cljs
+++ planck-cljs/test/planck/http_test.cljs
@@ -159,6 +159,15 @@
   (is (= "/foo" (:uri (do-request :get "/foo"))))
   (is (= "/foo" (:uri (do-request :post "/foo")))))

+(deftest request-json-body-test
+  (is (= "\"foo\"" (:body (do-request :post "/" {:content-type :json
+                                                 :accept :json
+                                                 :body "\"foo\""}))))
+  (is (= "5" (get-in (do-request :post "/" {:content-type :json
+                                            :accept :json
+                                            :body "\"foo\""})
+                   [:headers "content-length"]))))
+
 (deftest http-request-debug
   (let [url              (form-full-url "/")
         expected-request (fn [method]


Added documentation for a few newly added functions in core.

--- src/clojure/uncomplicate/neanderthal/core.clj
+++ src/clojure/uncomplicate/neanderthal/core.clj
@@ -128,8 +128,8 @@

   See uncomplicate.neanderthal.native, uncomplicate.neanderthal.opencl, etc.

-  (create-init cblas-single 3)
-  (create-init cblas-double 35 12)"
+  (create-raw cblas-single 3)
+  (create-raw cblas-double 35 12)"
   ([factory ^long n]
    (p/create-vector factory n
                     (.createDataSource (p/data-accessor factory) n)
@@ -140,7 +140,23 @@
                     nil)))

 (defn create-vector
-  ([factory source];;TODO documentation
+  "Creates a vector from source using the provided factory.
+
+  Accepts the following source:
+  - java.nio.ByteBuffer with a capacity divisible by the element width,
+  . which will be used as-is for backing the vector.
+  - a positive integer, which will be used as a dimension
+  . of new vector with zeroes as entries.
+  - a floating point number, which will be the only entry
+  . in a one-dimensional vector.
+  - a clojure sequence, which will be copied into a
+  . direct ByteBuffer that backs the new vector.
+  - varargs will be treated as a clojure sequence.
+
+  (create-vector *double-factory* [1 2 3])
+  (create-vector *single-factory* 12)
+  "
+  ([factory source]
    (let [acc ^DataAccessor (p/data-accessor factory)]
      (cond
        (integer? source) (create factory source)
@@ -157,6 +173,18 @@
    (create-vector factory (cons x xs))))

 (defn create-ge-matrix
+  "Creates a dense, column-oriented mxn matrix from source.
+
+  If called with two arguments, creates a zero matrix with dimensions mxn.
+
+  Accepts the following sources:
+  - java.nio.ByteBuffer with a capacity = elementWidth * m * n,
+  . which will be used as-is for backing the matrix.
+  - a clojure sequence, which will be copied into a
+  . direct ByteBuffer that backs the new vector.
+
+  (create-ge-matrix *double-factory* 15 13)
+  "
   ([factory m n source]
    (let [acc ^DataAccessor (p/data-accessor factory)]
      (cond


Allow different search result formats

Allow different search result formats

--- src/discuss/components/search/statements.cljs
+++ src/discuss/components/search/statements.cljs
@@ -15,7 +15,8 @@
 (s/def ::id integer?)
 (s/def ::text string?)
 (s/def ::name string?)
-(s/def ::nickname (s/keys :req-un [::name ::id]))
+(s/def ::nickname (s/or :eden (s/keys :req-un [::name ::id])
+                        :dbas string?))
 (s/def ::author (s/keys :req-un [::nickname]
                         :opt-un [::uid]))

@@ -93,7 +94,7 @@
   (let [identifier (:identifier statement)
         content (:content statement)]
     {:text (:text content)
-     :author {:nickname (:author content)}
+     :author {:nickname (:name (:author content))}
      :identifier {:aggregate-id (:aggregate-id identifier)
                   :entity-id (int (:entity-id identifier))
                   :version (:version identifier)}}))
@@ -172,7 +173,7 @@
                          (str (t :search :origin) ": " aggregator)
                          [:br]])
                       [:span.label.label-default
-                       (str (t :search :author) ": " (:name (:nickname author)))]]]]]))))
+                       (str (t :search :author) ": " (:nickname author))]]]]]))))
 (def result (om/factory Result))

 (defui Results


incrementing total-depth correctly by adding count of current path

--- src/babel/generate.cljc
+++ src/babel/generate.cljc
@@ -96,27 +96,25 @@
                                           (find-comp-paths-in bolt [:head])))
      bolts)))

-(defn add-all-comps-to-bolts-with-paths [bolts spec language-model total-depth comp-paths]
-  (log/trace (str "add-all-comps-to-bolts-with-paths with (empty? bolts): "
-                  (empty? bolts) "; spec: " (strip-refs spec)))
-  (cond (= spec :fail)
-        nil
-        true
-        (if (not (empty? comp-paths))
-          (let [path (first comp-paths)]
-            (add-all-comps-to-bolts-with-paths
-             (lazy-mapcat
-              (fn [bolt]
-                (log/debug (str "+comps: " ((:morph language-model) bolt) "@" "[" (string/join " " path) "]"))
-                (add-complement-to-bolt bolt path spec
-                                        language-model 0 total-depth
-                                        :max-total-depth max-total-depth))
-              bolts)
-             spec
-             language-model
-             total-depth
-             (rest comp-paths)))
-          bolts)))
+(defn add-all-comps-with-paths [bolts language-model total-depth comp-paths]
+  (log/trace (str "add-all-comps-with-paths with (empty? bolts): "
+                  (empty? bolts)))
+  (if (not (empty? comp-paths))
+    (let [path (first comp-paths)]
+      (add-all-comps-with-paths
+       (lazy-mapcat
+        (fn [bolt]
+          (log/debug (str "+comps: '" ((:morph language-model) bolt) "'@"
+                          "[" (string/join " " path) "]"
+                          "^" total-depth))
+          (add-complement-to-bolt bolt path
+                                  language-model 0 (+ total-depth (count path))
+                                  :max-total-depth max-total-depth))
+        bolts)
+       language-model
+       total-depth
+       (rest comp-paths)))
+    bolts))

 (defn find-comp-paths-in [bolt path]
   (if (not (= (get-in bolt path :none) :none))


Improve artifact retrieval error message

--- src/buildviz/gosync.clj
+++ src/buildviz/gosync.clj
@@ -253,8 +253,10 @@
       (catch JsonParseException e
         (log/errorf e "Unable to parse artifact list for %s" artifacts-url))
       (catch Exception e
-        (log/warn (format "Unable to get artifact list from %s, might have been deleted by Go" artifacts-url))
-        {}))))
+        (if-let [data (ex-data e)]
+          (log/errorf "Unable to get artifact list from %s (status %s): %s"
+                      artifacts-url (:status data) (:body data))
+          (log/errorf e "Unable to get artifact list from %s" artifacts-url))))))

 (defn xml-artifacts-for-job-run [job-instance]
   (let [file-tree (try-get-artifact-tree job-instance)]


Fix test redefinition

Fix test redefinition

--- test/cljam/bcf/t_writer.clj
+++ test/cljam/bcf/t_writer.clj
@@ -49,15 +49,13 @@

 (deftest about-encode-variant-indv
   (let [bb (#'bcf-writer/encode-variant-indv
-            {:genotype {0 [0 1] 1 [16 32] :n-sample 2}})]
+            {:genotype {0 [0 1] 1 [16 32]} :n-sample 2})]
     (is
      (= (seq (.array ^ByteBuffer bb))
         [0x11 0
          0x11 0 1
          0x11 1
-         0x11 16 32]))))
-
-(deftest about-encode-variant-indv
+         0x11 16 32])))
   (let [bb (#'bcf-writer/encode-variant-indv
             {:genotype {0 [[2 3] [4 3] [4 4]] 1 [48 48 43] 2 [1 8 5] 3 [[51 51] [51 51] [nil nil]]}
              :n-sample 3})]


Add secretbox binding with ByteBuffer out

--- src/caesium/binding.clj
+++ src/caesium/binding.clj
@@ -40,6 +40,12 @@
       ^long ^{LongLong {}} mlen
       ^bytes ^{Pinned {}} n
       ^bytes ^{Pinned {}} k]]
+    [^int crypto_secretbox_easy
+     [^java.nio.ByteBuffer ^{Pinned {}} c
+      ^bytes ^{Pinned {}} m
+      ^long ^{LongLong {}} mlen
+      ^bytes ^{Pinned {}} n
+      ^bytes ^{Pinned {}} k]]

     [^int crypto_secretbox_open_easy
      [^bytes ^{Pinned {}} m


Update protocols.cljc

Update protocols.cljc

--- src/cats/protocols.cljc
+++ src/cats/protocols.cljc
@@ -33,7 +33,7 @@
   "A marker protocol for identifying the valid context types."
   (-get-level [_] "Get a context priority level."))

- (defprotocol Contextual
+(defprotocol Contextual
   "Abstraction that establishes a concrete type as a member of a context.

   A great example es Maybe monad type Just. It implements


Fix misleading error message.

Fix misleading error message.

--- src/cljs/cljs/core.cljs
+++ src/cljs/cljs/core.cljs
@@ -7364,7 +7364,7 @@
   (-dispatch [mf args]))

 (defn- do-dispatch
-  [mf dispatch-fn args]
+  [mf name dispatch-fn args]
   (let [dispatch-val (apply dispatch-fn args)
         target-fn (-get-method mf dispatch-val)]
     (when-not target-fn
@@ -7415,7 +7415,7 @@
   (-methods [mf] @method-table)
   (-prefers [mf] @prefer-table)

-  (-dispatch [mf args] (do-dispatch mf dispatch-fn args))
+  (-dispatch [mf args] (do-dispatch mf name dispatch-fn args))

   IHash
   (-hash [this] (goog/getUid this)))


Remove 'arranged' attribute before rearranging

Prevents previously-arrange & condensed elements from
blocking un-condensing of parent elements.

--- src/cljs_formatter.cljs
+++ src/cljs_formatter.cljs
@@ -143,6 +143,7 @@
 (defn remove-all-styles! [elem]
   ;; remove-attr! doesn't always work
   (d/set-attr! elem :style "")
+  (d/remove-class! elem "arranged")
   (doseq [child (d/children elem)]
     (remove-all-styles! child)))


Fix stroke-cap, stroke-join and cursor functions. Close #170.

Some processingjs functions take use string constanats while we expected all of them to be int. For this functions we should not cast value to int but cast to string instead.

--- src/cljx/quil/core.cljx
+++ src/cljx/quil/core.cljx
@@ -1275,7 +1275,11 @@
   See cursor-image for specifying a generic image as the cursor
   symbol."
   ([] (.cursor (current-applet)))
-  ([cursor-mode] (.cursor (current-applet) (int (resolve-constant-key cursor-mode cursor-modes)))))
+  ([cursor-mode]
+   (let [cursor-mode (resolve-constant-key cursor-mode cursor-modes)]
+     (.cursor (current-applet)
+              #+clj (int cursor-mode)
+              #+cljs (str cursor-mode)))))

 (defn
   ^{:requires-bindings true
@@ -3976,7 +3980,9 @@
   parameters :square, :project, and :round. The default cap is :round."
   [cap-mode]
   (let [cap-mode (resolve-constant-key cap-mode stroke-cap-modes)]
-    (.strokeCap (current-graphics) (int cap-mode))))
+    (.strokeCap (current-graphics)
+                #+clj (int cap-mode)
+                #+cljs (str cap-mode))))

 (defn
   ^{:requires-bindings true
@@ -3994,7 +4000,9 @@
   renderers."
   [join-mode]
   (let [join-mode (resolve-constant-key join-mode stroke-join-modes)]
-    (.strokeJoin (current-graphics) (int join-mode))))
+    (.strokeJoin (current-graphics)
+                 #+clj (int join-mode)
+                 #+cljs (str join-mode))))

 (defn
   ^{:requires-bindings true


updates dependency versions (#473)

updates dependency versions (#473)

--- waiter/project.clj
+++ waiter/project.clj
@@ -30,30 +30,31 @@
                    :dev :dev
                    :perf (every-pred :perf (complement :explicit))}

-  :dependencies [[bidi "2.0.16"
+  :dependencies [[bidi "2.1.4"
                   :exclusions [prismatic/schema ring/ring-core]]
                  [twosigma/jet "0.7.10-20180627_133335-g2a9429e"]
                  [twosigma/clj-http "1.0.2-20180124_201819-gcdf23e5"
-                  :exclusions [commons-io org.clojure/tools.reader potemkin slingshot]]
-                 [clj-time "0.12.0"
+                  :exclusions [commons-codec commons-io org.clojure/tools.reader potemkin slingshot]]
+                 [clj-time "0.14.5"
                   :exclusions [joda-time]]
                  [com.google.guava/guava "20.0"]
-                 [com.taoensso/nippy "2.12.2"
+                 [com.taoensso/nippy "2.14.0"
                   :exclusions [org.clojure/clojure org.clojure/tools.reader]]
-                 [comb "0.1.0"
+                 [comb "0.1.1"
                   :exclusions [org.clojure/clojure]]
-                 [digest "1.4.5"]
-                 [fullcontact/full.async "0.9.0"
+                 [digest "1.4.8"
+                  :exclusions [org.clojure/clojure]]
+                 [fullcontact/full.async "1.0.0"
                   :exclusions [org.clojure/clojure org.clojure/core.async]]
-                 [jarohen/chime "0.1.9"
+                 [jarohen/chime "0.2.2"
                   :exclusions
                   [org.clojure/clojure
                    clj-time/clj-time
                    org.clojure/core.async]]
-                 [joda-time "2.9.4"]
+                 [joda-time "2.10"]
                  [twosigma/metrics-clojure "2.6.0-20180124_201441-g72cee16"
                   :exclusions [org.clojure/clojure io.netty/netty org.slf4j/slf4j-api]]
-                 [metrics-clojure-jvm "2.6.0"
+                 [metrics-clojure-jvm "2.10.0"
                   :exclusions [io.dropwizard.metrics/metrics-core
                                io.netty/netty
                                metrics-clojure
@@ -69,27 +70,28 @@
                  [org.apache.curator/curator-x-discovery "2.11.0"
                   :exclusions [io.netty/netty org.slf4j/slf4j-api]]
                  [org.clojure/clojure "1.8.0"]
-                 [org.clojure/core.async "0.3.442"
+                 [org.clojure/core.async "0.4.474"
                   :exclusions [org.clojure/clojure org.clojure/tools.reader]]
                  [org.clojure/core.memoize "0.7.1"
                   :exclusions [org.clojure/clojure]]
-                 [org.clojure/data.codec "0.1.0"]
+                 [org.clojure/data.codec "0.1.1"]
                  [org.clojure/data.json "0.2.6"]
-                 [org.clojure/data.priority-map "0.0.7"]
-                 [org.clojure/tools.cli "0.3.5"]
-                 [org.clojure/tools.logging "0.3.1"]
+                 [org.clojure/data.priority-map "0.0.10"]
+                 [org.clojure/tools.cli "0.4.1"]
+                 [org.clojure/tools.logging "0.4.1"]
                  [org.clojure/tools.namespace "0.2.11"]
-                 [org.clojure/tools.reader "0.10.0"]
-                 [org.slf4j/slf4j-log4j12 "1.7.21"
+                 [org.clojure/tools.reader "1.3.0"]
+                 [org.slf4j/slf4j-log4j12 "1.7.25"
                   :exclusions [log4j]]
-                 [potemkin "0.4.1"]
-                 [prismatic/plumbing "0.5.3"]
-                 [prismatic/schema "1.1.3"]
-                 [ring/ring-core "1.5.0"
+                 [potemkin "0.4.5"]
+                 [prismatic/plumbing "0.5.5"]
+                 [prismatic/schema "1.1.9"]
+                 [ring/ring-core "1.7.0"
                   :exclusions [org.clojure/tools.reader]]
                  [ring-basic-authentication "1.0.5"]
                  [slingshot "0.12.2"]
-                 [try-let "1.1.0"]]
+                 [try-let "1.2.0"
+                  :exclusions [org.clojure/clojure]]]

   :resource-paths ["resources"]
   :main waiter.main


[#2756] Add logic to keep flow group order

Lumen derive group order with columns data https://github.com/akvo/akvo-lumen/blob/issue-2756-data-groups/backend/src/akvo/lumen/lib/dataset.clj#L62

clojure core/group-by doesn't garantee any order

--- backend/src/akvo/lumen/lib/import/data_groups.clj
+++ backend/src/akvo/lumen/lib/import/data_groups.clj
@@ -22,7 +22,9 @@
 (defn- successful-execution [conn job-execution-id data-source-id dataset-id group-table-names columns {:keys [spec-name spec-description]} claims]
   (let [dataset-version-id (util/squuid)
         {:strs [rqg]} (env/all conn)
-        grouped-columns (group-by :groupId columns)]
+        grouped-columns (group-by :groupId columns)
+        ordered-groups-ids (distinct (map :groupId columns))
+        ordered-grouped-columns-col (reduce #(conj % [%2 (get grouped-columns %2)]) [] ordered-groups-ids)]
     (db.dataset-version/new-dataset-version-2 conn
                                               {:id dataset-version-id
                                                :dataset-id dataset-id
@@ -30,7 +32,7 @@
                                                :author claims
                                                :version 1
                                                :transformations []})
-    (doseq [[group-id cols] grouped-columns]
+    (doseq [[group-id cols] ordered-grouped-columns-col]
       (let [imported-table-name (util/gen-table-name "imported")
             table-name (get group-table-names group-id)
             instance-id-col (->> "metadata"


Changed default close operation to EXIT_ON_CLOSE for main GUI window

Signed-off-by: Daniel Borchmann <25857343a15bf1edafebc2912e82ecf775c585c1@mailbox.tu-dresden.de>

--- src/conexp/gui/base.clj
+++ src/conexp/gui/base.clj
@@ -48,7 +48,7 @@
   (let [main-frame (JFrame. "conexp-clj")]
     ;; main setup (including menu)
     (doto main-frame
-      (.setDefaultCloseOperation JFrame/DISPOSE_ON_CLOSE)
+      (.setDefaultCloseOperation JFrame/EXIT_ON_CLOSE)
       (.setSize 1000 800)
       (.setJMenuBar (JMenuBar.))
       (.setContentPane (JPanel. (BorderLayout.)))


lots of refactors around cogroup-by and names

--- src/datasplash/core.clj
+++ src/datasplash/core.clj
@@ -23,7 +23,7 @@
            [com.google.cloud.dataflow.sdk.util GcsUtil]
            [com.google.cloud.dataflow.sdk.util.common Reiterable]
            [com.google.cloud.dataflow.sdk.util.gcsfs GcsPath]
-           [com.google.cloud.dataflow.sdk.values KV PCollection TupleTag PBegin PCollectionList]
+           [com.google.cloud.dataflow.sdk.values KV PCollection TupleTag PBegin PCollectionList PInput]
            [java.io InputStream OutputStream DataInputStream DataOutputStream File]
            [java.net URI]
            [java.util UUID]
@@ -299,6 +299,40 @@
        tr))
    ptransform schema))

+(defprotocol ToApply
+  (tapply [pcoll nam tr]))
+
+(defrecord GroupSpecs [specs]
+  PInput
+  (expand [this] (map first specs))
+  (finishSpecifying [this] nil)
+  (getPipeline [this] (let [^PInput pval (-> specs (first) (first))]
+                        (.getPipeline pval))))
+
+(extend-protocol ToApply
+  PCollection
+  (tapply [pcoll nam tr] (if (and nam (not (empty? nam)))
+                           (.apply pcoll nam tr)
+                           (.apply pcoll tr)))
+  PCollectionList
+  (tapply [pcoll nam tr] (if (and nam (not (empty? nam)))
+                           (.apply pcoll nam tr)
+                           (.apply pcoll tr)))
+  PBegin
+  (tapply [pcoll nam tr] (if (and nam (not (empty? nam)))
+                           (.apply pcoll nam tr)
+                           (.apply pcoll tr)))
+  Pipeline
+  (tapply [pcoll nam tr] (if (and nam (not (empty? nam)))
+                           (.apply pcoll nam tr)
+                           (.apply pcoll tr)))
+  KeyedPCollectionTuple
+  (tapply [pcoll nam tr] (if (and nam (not (empty? nam)))
+                           (.apply pcoll nam tr)
+                           (.apply pcoll tr)))
+  GroupSpecs
+  (tapply [group-specs _ tr] (.apply tr group-specs)))
+
 (defn apply-transform
   "apply the PTransform to the given Pcoll applying options according to schema."
   [pcoll ^PTransform transform schema
@@ -306,9 +340,7 @@
   (let [nam (some-> options (:name) (name))
         clean-opts (dissoc options :name :coder :coll-name)
         configured-transform (with-opts schema clean-opts transform)
-        bound (if (and nam (not (empty? nam)))
-                (.apply pcoll nam configured-transform)
-                (.apply pcoll configured-transform))]
+        bound (tapply pcoll nam configured-transform)]
     (-> bound
         (cond-> coder (.setCoder coder))
         (cond-> coll-name (.setName coll-name)))))
@@ -956,8 +988,6 @@
    :added "0.1.0"}
   ([from options p]
    (let [opts (assoc options
-                     :label (str "read-text-file-from"
-                                 (clean-filename from))
                      :coder (or
                              (:coder options)
                              (StringUtf8Coder/of)))]
@@ -967,16 +997,6 @@
                           (merge named-schema text-reader-schema) opts))))
   ([from p] (read-text-file from {} p)))

-(defn- read-edn-file-transform
-  [from options]
-  (let [safe-opts (dissoc options :name)]
-    (ptransform
-     :read-edn-file
-     [p]
-     (->> p
-          (read-text-file from (dissoc options :coder))
-          (from-edn options)))))
-
 (defn read-edn-file
   {:doc (with-opts-docstr "Reads a PCollection of edn strings from disk or Google Storage, with records separated by newlines.

@@ -990,22 +1010,16 @@
    :added "0.1.0"}
   ([from options ^Pipeline p]
    (let [opts (assoc options
-                     :label (str "read-edn-file-from-"
-                                 (clean-filename from))
                      :coder (or (:coder options) (make-nippy-coder)))]
-     (apply-transform p (read-edn-file-transform from opts)
-                      (merge base-schema text-reader-schema) opts)))
+     (pt->>
+      (or (:name options) (str "read-text-file-from"
+                               (clean-filename from)))
+      p
+      (read-text-file from (-> options
+                               (dissoc :coder)
+                               (assoc :name "read-text-file")))
+      (from-edn (assoc options :name "parse-edn")))))
   ([from p] (read-edn-file from {} p)))
-
-(defn- write-edn-file-transform
-  [to options]
-  (let [safe-opts (dissoc options :name :coder)]
-    (ptransform
-     :write-edn-file
-     [^PCollection pcoll]
-     (->> pcoll
-          (to-edn safe-opts)
-          (write-text-file to safe-opts)))))

 (defn write-edn-file
   {:doc (with-opts-docstr
@@ -1020,32 +1034,18 @@
           base-schema text-writer-schema)
    :added "0.1.0"}
   ([to options ^PCollection pcoll]
-   (let [opts (assoc options :label (str "write-edn-file-to-" (clean-filename to))
-                     :coder nil)]
-     (apply-transform pcoll (write-edn-file-transform to opts)
-                      named-schema opts)))
+   (let [opts (assoc options :coder nil)]
+     (pt->>
+      (or (:name options) (str "write-edn-file-to-" (clean-filename to)))
+      pcoll
+      (to-edn (assoc options :name "encode-edn"))
+      (write-text-file to (assoc options :name "write-file")))))
   ([to pcoll] (write-edn-file to {} pcoll)))

 (def json-reader-schema
   {:key-fn {:docstr "Selects a policy for parsing map keys. If true, keywordizes the keys. If given a fn, uses it to transform each map key."}
    :return-type {:docstr "Allows passing in a function to specify what kind of types to return."}})

-(defn- read-json-file-transform
-  [from {:keys [key-fn return-type] :as options}]
-  (let [safe-opts (dissoc options :name :key-fn :return-type)]
-    (ptransform
-     :read-json-file
-     [p]
-     (->> p
-          (read-text-file from (dissoc safe-opts :coder))
-          (dmap (fn [l]
-                  (cond
-                    (and key-fn return-type) (json/decode l key-fn return-type)
-                    key-fn (json/decode l key-fn)
-                    return-type (json/decode l nil return-type)
-                    :else (json/decode l)))
-                safe-opts)))))
-
 (defn read-json-file
   {:doc (with-opts-docstr "Reads a PCollection of JSON strings from disk or Google Storage, with records separated by newlines.

@@ -1057,13 +1057,24 @@
 ```"
           base-schema text-reader-schema json-reader-schema)
    :added "0.2.0"}
-  ([from options ^Pipeline p]
+  ([from {:keys [key-fn return-type] :as options} ^Pipeline p]
    (let [opts (assoc options
                      :label (str "read-json-file-from-"
                                  (clean-filename from))
                      :coder (or (:coder options) (make-nippy-coder)))]
-     (apply-transform p (read-json-file-transform from opts)
-                      (merge base-schema text-reader-schema) opts)))
+     (pt->>
+      (or (:name options) ("read-json-file-from-" (clean-filename from)))
+      p
+      (read-text-file from (-> options
+                               (dissoc :coder)
+                               (assoc :name "read-text-file")))
+      (dmap (fn [l]
+              (cond
+                (and key-fn return-type) (json/decode l key-fn return-type)
+                key-fn (json/decode l key-fn)
+                return-type (json/decode l nil return-type)
+                :else (json/decode l)))
+            (assoc options :name "json-decode")))))
   ([from p] (read-json-file from {} p)))

 (def json-writer-schema
@@ -1095,10 +1106,14 @@
           base-schema text-writer-schema json-writer-schema)
    :added "0.2.0"}
   ([to options ^PCollection pcoll]
-   (let [opts (assoc options :label (str "write-json-file-to-" (clean-filename to))
-                     :coder nil)]
-     (apply-transform pcoll (write-json-file-transform to opts)
-                      named-schema opts)))
+   (let [json-opts (select-keys options (keys json-writer-schema))]
+     (pt->>
+      (or (:name options) "write-json-file")
+      pcoll
+      (dmap (fn [l] (json/encode l json-opts)) (assoc options
+                                                      :name "encode-json"
+                                                      :coder (StringUtf8Coder/of)))
+      (write-text-file to (assoc options :name "write-text-file")))))
   ([to pcoll] (write-json-file to {} pcoll)))

 (defn make-partition-mapping
@@ -1145,6 +1160,7 @@
 ;;;;;;;;;;;

 (defn make-keyed-pcollection-tuple
+  ^KeyedPCollectionTuple
   [pcolls]
   (let [empty-kpct (KeyedPCollectionTuple/empty (.getPipeline (first pcolls)))]
     (reduce
@@ -1153,6 +1169,20 @@
              new-coll-tuple (.and coll-tuple tag pcoll)]
          new-coll-tuple))
      empty-kpct (map-indexed (fn [idx v] [idx v]) pcolls))))
+
+(defn make-group-specs
+  [specs]
+  (if (instance? GroupSpecs specs)
+    specs
+    (let [safe-specs (into []
+                           (doall
+                            (for [[pcoll key-fn spec] specs]
+                              (cond
+                                (nil? spec) [pcoll key-fn {:type :optional}]
+                                (keyword? spec) [pcoll key-fn {:type spec}]
+                                (map? spec) [pcoll key-fn spec]
+                                :else (throw (ex-info "Invalid spec for cogroup" {:specs specs}))))))]
+      (GroupSpecs. safe-specs))))

 (defn greedy-read-cogbkresult
   [raw-values tag]
@@ -1174,44 +1204,67 @@
         (recur it)))))

 (defn cogroup-transform
-  ([specs {:keys [join-nil?] :as options}]
+  ([{:keys [join-nil?] :as options}]
    (let [opts (assoc options :label :raw-cogroup)]
      (ptransform
       :cogroup
-      [^KeyedPCollectionTuple pcolltuple]
-      (let [ordered-tags (->> pcolltuple
+      [^GroupSpecs group-specs]
+      (let [root-name (or (:name options) "cogroup")
+            pcolls (for [[idx [pcoll f {:keys [drop-nil?] :as opts}]]
+                         (map-indexed (fn [idx s] (if (instance? PCollection s)
+                                                    [idx [s nil nil]] [idx s])) (:specs group-specs))]
+                     (let [local-name (str root-name "-" (.getName pcoll))
+                           op (if f
+                                (with-keys f {:name (str local-name "-group-by")} pcoll)
+                                pcoll)]
+                       (if drop-nil?
+                         (dfilter (if f
+                                    (fn [^KV kv] (not (nil? (.getKey kv))))
+                                    (fn [v] (not (nil? v))))
+                                  {:name (str local-name "-drop-nil")
+                                   :without-coercion-to-clj true}
+                                  op)
+                         op)))
+            pcolltuple (make-keyed-pcollection-tuple pcolls)
+            ordered-tags (->> pcolltuple
                               (.getKeyedCollections)
                               (map #(.getTupleTag %))
                               (sort-by #(.getId %)))
             rel (apply-transform pcolltuple (CoGroupByKey/create) base-schema opts)
+            required-set (->> (:specs group-specs)
+                              (map-indexed (fn [idx [_ _ {:keys [type]}]]
+                                             (when (= type :required) idx)))
+                              (remove nil?)
+                              (into #{}))
+            required-count (count required-set)
             final-rel (pardo
                        (fn [^DoFn$ProcessContext c]
                          (let [^KV kv (.element c)
                                k (.getKey kv)
-                               ^CoGbkResult raw-values (.getValue kv)
-                               required-list (->> specs
-                                                  (map-indexed
-                                                   (fn [idx {:keys [type]}]
-                                                     (when (= type :required) idx)))
-                                                  (remove nil?))
-                               required-count (count required-list)]
+                               ^CoGbkResult raw-values (.getValue kv)]
                            ;; skip if a required part of the group is empty
                            (when-not (some identity
-                                           (map (fn [tag {:keys [type]}]
-                                                  (if (= type :required)
-                                                    (not (.hasNext (.iterator (.getAll raw-values tag))))
-                                                    false))
-                                                ordered-tags specs))
+                                           (map-indexed (fn [idx tag]
+                                                          (if (required-set idx)
+                                                            (not (-> (.getAll raw-values tag)
+                                                                     (.iterator)
+                                                                     (.hasNext)))
+                                                            false))
+                                                        ordered-tags))
                              (if (and (not join-nil?) (nil? k))
                                (cond
-                                 (= required-count 0) (doseq [[idx tag] (map-indexed (fn [idx tag] [idx tag]) ordered-tags)]
-                                                        (greedy-emit-cogbkresult raw-values (count ordered-tags) idx tag c))
-                                 (= required-count 1) (greedy-emit-cogbkresult raw-values
-                                                                               (count ordered-tags)
-                                                                               (first required-list)
-                                                                               (nth ordered-tags
-                                                                                (first required-list))
-                                                                               c)
+                                 (= required-count 0) (doseq [[idx tag]
+                                                              (map-indexed
+                                                               (fn [idx tag] [idx tag]) ordered-tags)]
+                                                        (greedy-emit-cogbkresult
+                                                         raw-values (count ordered-tags) idx tag c))
+                                 (= required-count 1) (greedy-emit-cogbkresult
+                                                       raw-values
+                                                       (count ordered-tags)
+                                                       (first required-set)
+                                                       (nth ordered-tags
+                                                            (first required-set))
+                                                       c)
                                  :else nil)
                                (let [values (mapv
                                              (fn [tag]
@@ -1219,11 +1272,11 @@
                                              ordered-tags)]
                                  (.output c (make-kv k values)))))))
                        (assoc opts
+                              :name (str root-name "-apply-requirements")
                               :without-coercion-to-clj true
                               :coder (make-kv-coder (.getKeyCoder (.getCoder rel))
                                                     (make-nippy-coder)))
                        rel)]
-
         final-rel)))))

 (defn cogroup
@@ -1234,8 +1287,8 @@

 Example:
 ```
-(let [pcoll1 (ds/generate [[:a 11] [:b 12] [:a 13]])
-      pcoll2 (ds/generate [[:a 21] [:c 22]])]
+(let [pcoll1 (ds/generate-input [[:a 11] [:b 12] [:a 13]])
+      pcoll2 (ds/generate-input [[:a 21] [:c 22]])]
   (ds/cogroup [{:type :required} {:type :optional}] {:name :my-cogroup} [pcoll1 pcoll2]))
 ;; returns:
 ;; '([:a ['(11 13) '(21)]]
@@ -1243,18 +1296,28 @@
 ;;   [:c [nil '(22)]])

 ```
-
 See https://cloud.google.com/dataflow/java-sdk/JavaDoc/com/google/cloud/dataflow/sdk/transforms/join/CoGroupByKey and for a more idiomatic approach to joins [[cogroup-by]] and [[join-by]]"
           base-schema)
    :added "0.1.0"}
-  [specs options pcolls]
-  (let [opts (assoc options :label :cogroup)
-        pcolltuple (make-keyed-pcollection-tuple pcolls)]
-    (apply-transform pcolltuple (cogroup-transform specs opts) base-schema opts)))
+  ([specs options]
+   (let [opts (assoc options :label :cogroup)]
+     (apply-transform (make-group-specs specs) (cogroup-transform opts) named-schema opts)))
+  ([specs] (cogroup specs nil)))
+
+(defn cogroup-by-transform
+  [options reduce-fn]
+  (ptransform
+   :cogroup-by
+   [^GroupSpecs group-specs]
+   (let [grouped-colls (cogroup group-specs options)
+         root-name (or (:name options) "cogroup-by")]
+     (if reduce-fn
+       (dmap reduce-fn (assoc options :name (str root-name "-collector")) grouped-colls)
+       grouped-colls))))

 (defn cogroup-by
   {:doc (with-opts-docstr
-          "Takes a specification of the join between pcolls, and returns a PCollection of KVs with values being list of values corresponding to the key-fn. The specification is a list of triple
+          "Takes a specification of the join between pcolls, a function producing the key on which to join and returns a PCollection of KVs with values being list of values corresponding to the key-fn. The specification is a list of triple.

  Only one option is supported:

@@ -1265,43 +1328,30 @@
 (ds/cogroup [{:type :required} {:type :optional}] {:name :my-cogroup} [pcoll1 pcoll2])

 ```
-
 See https://cloud.google.com/dataflow/java-sdk/JavaDoc/com/google/cloud/dataflow/sdk/transforms/join/CoGroupByKey and for a more idiomatic approach to joins [[cogroup-by]] and [[join-by]]"
-          base-schema)
+          named-schema)
    :added "0.1.0"}
   ([options specs reduce-fn]
-   (let [safe-opts (dissoc options :name)
-         pcolls (for [[idx [pcoll f {:keys [drop-nil?]}]] (map-indexed vector specs)]
-                  (let [op (with-keys f (assoc safe-opts :name (str "group-by-" idx)) pcoll)]
-                    (if drop-nil?
-                      (dfilter (fn [^KV kv] (not (nil? (.getKey kv))))
-                               {:name (str "drop-nil-" idx)
-                                :without-coercion-to-clj true}
-                               op)
-                      op)))
-         safe-specs (into []
-                          (doall
-                           (for [[pcoll key-fn spec] specs]
-                             (cond
-                               (nil? spec) {:type :optional}
-                               (keyword? spec) {:type spec}
-                               (map? spec) spec
-                               :else (throw (ex-info "Invalid spec for cogroup" {:specs specs}))))))
-         grouped-coll (cogroup safe-specs safe-opts pcolls)]
-     (if reduce-fn
-       (dmap reduce-fn safe-opts grouped-coll)
-       grouped-coll)))
+   (let [group-specs (make-group-specs specs)
+         cogroup-by-tr (cogroup-by-transform options reduce-fn) ]
+     (apply-transform group-specs cogroup-by-tr named-schema options)))
   ([options specs] (cogroup-by options specs nil)))

 (defn join-by
   ([options specs join-fn]
-   (->> (cogroup-by options specs)
-        (dmapcat (fn [^KV kv]
-                   (let [results (.getValue kv)
-                         results-ok (map #(if (empty? %) [nil] %) results)
-                         raw-res (apply combo/cartesian-product results-ok)
-                         res (map (fn [prod] (apply join-fn prod)) raw-res)]
-                     res)) {:without-coercion-to-clj true })))
+   (let [nam (or (:name options) "join-by")]
+     (pt->>
+      (or (:name options) "join-by")
+      (cogroup-by (assoc options :name (str nam "-cogroup-by")) specs)
+      (dmapcat (fn [^KV kv]
+                 (let [results (.getValue kv)
+                       results-ok (map #(if (empty? %) [nil] %) results)
+                       raw-res (apply combo/cartesian-product results-ok)
+                       res (map (fn [prod] (apply join-fn prod)) raw-res)]
+                   res))
+               {:name (let [root-name (or (:name options) "join-by")]
+                        (str root-name "-cartesian-product"))
+                :without-coercion-to-clj true }))))
   ([specs join-fn] (join-by {} specs join-fn)))

 (defn ddistinct


Make the tests pass

Make the tests pass

--- src/desdemona/query.clj
+++ src/desdemona/query.clj
@@ -36,5 +36,7 @@
    `(l/featurec ~lvar {~attr ~value})))

 (defn run-dsl-query
-  [dsl-query events]
-  (run-logic-query (dsl->logic dsl-query) events))
+  ([dsl-query events]
+   (run-logic-query (dsl->logic dsl-query) events))
+  ([n-answers dsl-query events]
+   (run-logic-query n-answers (dsl->logic dsl-query) events)))


Set default issue to an english one

Set default issue to an english one

--- src/discuss/config.cljs
+++ src/discuss/config.cljs
@@ -5,7 +5,7 @@
 (def user "Q2hyaXN0aWFu")

 (def api {:host  "http://localhost:4284/"
-          :init  "api/elektroautos"
+          :init  "api/cat-or-dog"
           :base  "api/"
           :login "api/login"
           :add   {:add-start-statement "api/add/start_statement"


added safe-max function to safely compute the maximum value of a numeric sequence, which is potentially empty

--- src/distillery/data.clj
+++ src/distillery/data.clj
@@ -166,6 +166,12 @@
 ;; ## Collection Math
 ;; A few helper functions to simplify some mathematical tasks on collections.

+(defn safe-max
+  "Computes the maximum of a numeric sequence.
+  Returns `0` if the sequence is empty."
+  [xs]
+  (if (empty? xs) 0 (apply max xs)))
+
 (defn sum
   "Computes the sum of a numeric sequence."
   [xs] (apply + xs))
@@ -180,7 +186,5 @@
 (defn squared-sum
   "Computes the sum of the squared items of a sequence."
   [xs] (apply + (map #(* % %) xs)))
-
-


CMR-8464: Added handling of collection-concept-ids in concept-id parameter value.

--- search-app/src/cmr/search/services/parameters/conversion.clj
+++ search-app/src/cmr/search/services/parameters/conversion.clj
@@ -2,7 +2,7 @@
   "Contains functions for parsing and converting query parameters to query conditions"
   (:require
    [clojure.set :as set]
-   [clojure.string :as str]
+   [clojure.string :as string]
    [cmr.common-app.services.search.group-query-conditions :as gc]
    [cmr.common-app.services.search.parameters.converters.nested-field :as nf]
    [cmr.common-app.services.search.params :as common-params]
@@ -207,7 +207,7 @@

 (defmethod common-params/parameter->condition :keyword
   [_ _ _ value _]
-  (cqm/text-condition :keyword (str/lower-case value)))
+  (cqm/text-condition :keyword (string/lower-case value)))

 (def collection-only-params
   "List of parameters that are valid in collection query models, but not in granule query models."
@@ -265,7 +265,7 @@
   (let [case-sensitive? (common-params/case-sensitive-field? concept-type param options)
         pattern? (common-params/pattern-field? concept-type param options)
         group-operation (common-params/group-operation param options :and)
-        target-field (keyword (str/replace (name param) #"-h$" ""))]
+        target-field (keyword (string/replace (name param) #"-h$" ""))]

     (if (map? (first (vals value)))
       ;; If multiple variables are passed in like the following
@@ -334,11 +334,11 @@
        (map #(common-params/parameter->condition context concept-type param % options) value))
       (gc/or-conds
        (map #(common-params/parameter->condition context concept-type param % options) value)))
-    (let [[start-date end-date] (map str/trim (str/split value #","))]
+    (let [[start-date end-date] (map string/trim (string/split value #","))]
       (cqm/map->DateRangeCondition
        {:field param
-        :start-date (when-not (str/blank? start-date) (parser/parse-datetime start-date))
-        :end-date (when-not (str/blank? end-date) (parser/parse-datetime end-date))}))))
+        :start-date (when-not (string/blank? start-date) (parser/parse-datetime start-date))
+        :end-date (when-not (string/blank? end-date) (parser/parse-datetime end-date))}))))

 (defmethod common-params/parameter->condition :readable-granule-name
   [context concept-type param value options]
@@ -358,15 +358,15 @@
   [_ _ _ value _]
   (if (= "unset" value)
     cqm/match-all
-    (qm/->HasGranulesCondition (= "true" (str/lower-case value)))))
+    (qm/->HasGranulesCondition (= "true" (string/lower-case value)))))

 (defmethod common-params/parameter->condition :has-granules-or-cwic
   [_ _ _ value _]
-  (qm/->HasGranulesOrCwicCondition (= "true" (str/lower-case value))))
+  (qm/->HasGranulesOrCwicCondition (= "true" (string/lower-case value))))

 (defmethod common-params/parameter->condition :has-granules-or-opensearch
   [_ _ _ value _]
-  (qm/->HasGranulesOrOpenSearchCondition (= "true" (str/lower-case value))))
+  (qm/->HasGranulesOrOpenSearchCondition (= "true" (string/lower-case value))))

 (defn- collection-data-type-matches-science-quality?
   "Convert the collection-data-type parameter with wildcards to a regex. This function
@@ -374,11 +374,11 @@
   the special case of SCIENCE_QUALITY."
   [param case-sensitive]
   (let [param (if case-sensitive
-                (str/upper-case param)
+                (string/upper-case param)
                 param)
         pattern (-> param
-                    (str/replace #"\?" ".")
-                    (str/replace #"\*" ".*")
+                    (string/replace #"\?" ".")
+                    (string/replace #"\*" ".*")
                     re-pattern)]
     (re-find pattern "SCIENCE_QUALITY")))

@@ -393,7 +393,7 @@
     (let [case-sensitive (common-params/case-sensitive-field? concept-type :collection-data-type options)
           pattern (common-params/pattern-field? concept-type :collection-data-type options)]
       (if (or (= "SCIENCE_QUALITY" value)
-              (and (= "SCIENCE_QUALITY" (str/upper-case value))
+              (and (= "SCIENCE_QUALITY" (string/upper-case value))
                    (not= "false" (get-in options [:collection-data-type :ignore-case])))
               (and pattern
                    (collection-data-type-matches-science-quality? value case-sensitive)))
@@ -457,14 +457,14 @@
                                   [:has-granules-created-at])
                                 (when (:has-granules-revised-at params)
                                   [:has-granules-revised-at])
-                                (when-not (str/blank? (:include-tags params))
+                                (when-not (string/blank? (:include-tags params))
                                   [:tags])
                                 ;; Always include temporal, the processor will see if any temporal
                                 ;; conditions exist
                                 [:temporal-conditions])
         keywords (when (:keyword params)
-                   (str/split (str/lower-case (:keyword params)) #" "))
-        params (if keywords (assoc params :keyword (str/join " " keywords)) params)
+                   (string/split (string/lower-case (:keyword params)) #" "))
+        params (if keywords (assoc params :keyword (string/join " " keywords)) params)
         simplify-shapefile? (= "true" (:simplify-shapefile params))]
     [(dissoc params
              :boosts :include-granule-counts :include-has-granules :include-facets :echo-compatible
@@ -477,8 +477,8 @@
                  :all-revisions? (= "true" (:all-revisions params))
                  :simplify-shapefile? (when simplify-shapefile? simplify-shapefile?)
                  :facets-size (:facets-size params)
-                 :result-options (merge (when-not (str/blank? (:include-tags params))
-                                          {:tags (map str/trim (str/split (:include-tags params) #","))})
+                 :result-options (merge (when-not (string/blank? (:include-tags params))
+                                          {:tags (map string/trim (string/split (:include-tags params) #","))})
                                         (when (or begin-tag end-tag snippet-length num-snippets)
                                           {:highlights
                                            {:begin-tag begin-tag
@@ -494,19 +494,22 @@
         result-features (when (= "v2" (util/safe-lowercase (:include-facets params)))
                           [:facets-v2])
         regular-params (dissoc params :echo-compatible :include-facets :simplify-shapefile)
-        concept-ids (:concept-id regular-params)
-        num-concept-ids (if (sequential? concept-ids)
-                          (count concept-ids)
-                          1)
         {:keys [page-size offset]} query-attribs
-        only-concept-id-params? (-> granule-param-names
-                                    (some (keys (dissoc regular-params :concept-id)))
-                                    nil?)
+        concept-id (:concept-id regular-params)
+        concept-ids (when concept-id
+                      (if (sequential? concept-id)
+                          concept-id
+                          [concept-id]))
+        only-concept-id-params? (nil?
+                                 (some granule-param-names
+                                       (keys (dissoc regular-params :concept-id))))
+        no-collection-in-concept-id? (nil? (some #(string/starts-with? % "C") concept-ids))
         gran-specific-items-query? (if (and (:concept-id regular-params)
                                             only-concept-id-params?
+                                            no-collection-in-concept-id?
                                             (= 0 offset)
                                             (or (= page-size :unlimited)
-                                                (>= page-size num-concept-ids)))
+                                                (>= page-size (count concept-ids))))
                                      true
                                      false)]
     [regular-params


Document the legacy behavior in core.clj

Prepare to replace json with database.

--- src/flo_invest/core.clj
+++ src/flo_invest/core.clj
@@ -5,13 +5,21 @@
     (:use clojure-csv.core)
     )

-(defn parse-dir [datadir]
+(defn parse-dir
+  "Iterate through the directory datadir and find triples of files
+  with stock information.
+
+  Returns a seq containing triples of maps {:isin :type :file},
+  where :isin is an isin string, :type is one
+  of :balancesheet, :incomestatement or :keyratios and file is a
+  handle to the file.
+  "
+  [datadir]
   (group-by :isin
             (for [f (file-seq (io/file datadir))
                   :let [filename (.getName f)
                         parts (clojure.string/split filename #" ")
-                        isin (parts 0)
-                        ]
+                        isin (parts 0)]
                   :when (and
                          (.endsWith filename ".csv")
                          (= 3 (count parts))
@@ -80,11 +88,14 @@
   (merge {:tangible_book_value
           (minus (input-map :reported_book_value)
                  (divide (plus (with-scale (input-map :goodwill) 10) (input-map :intangibles))
-                         (input-map :shares_outstanding)
-                         ))
+                         (input-map :shares_outstanding)))
           } input-map))

-(defn load-data [file-data]
+(defn load-data
+  "Process files as delivered by parse-dir.
+  Returns output in format heavily inspired by the previous python format.
+  "
+  [file-data]
   (let [
         missing (as-money "" "EUR")
         inputs  {:non_redeemable_preferred_stock 0.0
@@ -104,8 +115,7 @@
         keyratios (slurp-csv :keyratios file-data-set)
         ]
     (add-tangible-book-value (apply merge (cons inputs
-                                                 (concat
-                                                  (parse-income income)
-                                                  (parse-balance balance)
-                                                  (parse-keyratios keyratios)
-                                                  ))))))
+                                                (concat
+                                                 (parse-income income)
+                                                 (parse-balance balance)
+                                                 (parse-keyratios keyratios)))))))


(TK-369) As jetty no longer checks for isStarted, we no longer do as well

--- src/puppetlabs/trapperkeeper/services/webserver/normalized_uri_helpers.clj
+++ src/puppetlabs/trapperkeeper/services/webserver/normalized_uri_helpers.clj
@@ -59,14 +59,8 @@
   (proxy [HandlerWrapper] []
     (handle [^String target ^Request base-request
              ^HttpServletRequest request ^HttpServletResponse response]
-      (let [handler (proxy-super getHandler)
-            is-started (proxy-super isStarted)]
-        ;; It may not strictly be necessary to check if the wrapping handler
-        ;; is started in order to let a request through but that's what the
-        ;; base `HandlerWrapper` class from Jetty does, so it seemed best to
-        ;; follow the pattern:
-        ;; https://github.com/eclipse/jetty.project/blob/jetty-9.2.10.v20150310/jetty-server/src/main/java/org/eclipse/jetty/server/handler/HandlerWrapper.java#L95
-        (when (and handler is-started)
+      (let [handler (proxy-super getHandler)]
+        (when handler
           (if-let [normalized-uri
                    (try
                      (normalize-uri-path request)


comments and cleanup

comments and cleanup


Formatting

Formatting

--- src/kixi/hecuba/web.clj
+++ src/kixi/hecuba/web.clj
@@ -9,6 +9,7 @@
    [clojure.tools.logging :refer :all]
    [clojure.edn :as edn]
    [liberator.core :refer (resource defresource)]
+   [kixi.hecuba.hash :refer (sha1)]
    [kixi.hecuba.kafka :as kafka]
    [kixi.hecuba.protocols :refer (upsert! item items)]
    )
@@ -42,8 +43,9 @@
                  "text/html" projects ; do something here to render the href as a link
                  projects))
   :post! (fn [{{body :body} :request}]
-          (let [payload (io! (edn/read (java.io.PushbackReader. (io/reader body))))]
-            (upsert! commander payload))))
+           (let [payload (io! (edn/read (java.io.PushbackReader. (io/reader body))))]
+             (upsert! commander payload))))
+
 (defresource project-resource [querier]
   :allowed-methods #{:get}
   :available-media-types base-media-types


allows keywords as localStorage keys and updates method call to new syntax

--- src/lib/cljs/local.cljs
+++ src/lib/cljs/local.cljs
@@ -6,7 +6,7 @@

 (defn get-item [k]
   (if (local-storage?)
-    (. js/localStorage (getItem k))))
+    (.getItem js/localStorage (if (keyword? k) (name k) k))))

 (defn get-clojure [k]
   (if-let [item (get-item k)]
@@ -15,7 +15,7 @@
 (defn set-item! [k v]
   (if (local-storage?)
     (let [item (if (string? v) v (pr-str v))]
-      (.setItem js/localStorage k item))))
+      (.setItem js/localStorage (if (keyword? k) (name k) k) item))))

 (defn conj-item!
   "Assume localStorage for key k is (or should become) a vector and conj value v"


set starting clicks to 0

set starting clicks to 0

--- src/node/game/core.cljs
+++ src/node/game/core.cljs
@@ -354,14 +354,14 @@
                        :hand (zone :hand (take 5 corp-deck))
                        :discard [] :scored [] :rfg [] :play-area []
                        :servers {:hq {} :rd{} :archives {} :remote []}
-                       :click 3 :credit 5 :bad-publicity 0 :agenda-point 0 :max-hand-size 5
+                       :click 0 :credit 5 :bad-publicity 0 :agenda-point 0 :max-hand-size 5
                        :click-per-turn 3 :agenda-point-req 7 :keep false}
                 :runner {:user (:user runner) :identity runner-identity
                          :deck (zone :deck (drop 5 runner-deck))
                          :hand (zone :hand (take 5 runner-deck))
                          :discard [] :scored [] :rfg [] :play-area []
                          :rig {:program [] :resource [] :hardware []}
-                         :click 4 :credit 5 :memory 4 :link 0 :tag 0 :agenda-point 0 :max-hand-size 5
+                         :click 0 :credit 5 :memory 4 :link 0 :tag 0 :agenda-point 0 :max-hand-size 5
                          :hq-access 1 :rd-access 1
                          :brain-damage 0 :click-per-turn 4 :agenda-point-req 7 :keep false}})]
     (card-init state :corp corp-identity)


fix the key queue

fix the key queue

--- src-cljs/frontend/core.cljs
+++ src-cljs/frontend/core.cljs
@@ -62,7 +62,7 @@
 (defn track-key-state [cast! direction suppressed-key-combos event]
   (let [key-set (keyq/event->key event)]
     (when-not (or (contains? #{"input" "textarea"} (string/lower-case (.. event -target -tagName)))
-                  (.. event -target -contentEditable))
+                  (= "true" (.. event -target -contentEditable)))
       (when (contains? suppressed-key-combos key-set)
         (.preventDefault event))
       (when-not (.-repeat event)


Fix parens in docstring example

Fix parens in docstring example

--- src-cljs/frontend/utils.clj
+++ src-cljs/frontend/utils.clj
@@ -128,13 +128,13 @@
       [:title :author :content])
     Object
     (render [this]
-      (let [{:keys [title author content]} (om/props)])
-      (component Post
-        (html
-         [:article
-          [:h1 title]
-          [:h2 \"by \" author]
-          [:div.body content]]))))"
+      (let [{:keys [title author content]} (om/props)]
+        (component Post
+          (html
+           [:article
+            [:h1 title]
+            [:h2 \"by \" author]
+            [:div.body content]])))))"
   [name body]
   (assert (and (symbol? name)
                (nil? (namespace name)))


Consolidated view.

Consolidated view.

--- src-cljs/tea_time/core.cljs
+++ src-cljs/tea_time/core.cljs
@@ -11,12 +11,117 @@
   (:import goog.History))

+(re-frame/register-handler
+  :process-teas-response
+  (fn [app-db [_ response]]
+    (assoc-in app-db [:teas-list] response)))
+
+(re-frame/register-handler
+  :process-teas-bad-response
+  (fn [app-db [_ response]]
+    (println "ERROR: " response)
+    app-db))
+
+(re-frame/register-handler
+  :load-teas
+  (fn [app-db _]
+    (ajax/GET "/api/teas"
+              {:handler #(re-frame/dispatch [:process-teas-response %1])
+               :error-handler #(re-frame/dispatch [:process-teas-bad-response %1])
+               :response-format :json
+               :keywords? true})
+    app-db))
+
+(re-frame/register-handler
+  :delete-tea
+  (fn [db [_ tea-name]]
+    (ajax/GET (str "/api/teas/" tea-name "/delete")
+              :error-handler (fn [response]
+                               (println "DELETE-TEA ERROR" response))
+              :handler (fn [response]
+                         (re-frame/dispatch [:load-teas])))))
+
+(re-frame/register-handler
+  :delete-tea
+  (fn [app-db [_ id]]
+    (ajax/GET (str "/api/teas/" id "/delete")
+              :error-handler (fn [response] (println "DELETE-TEA ERROR " response))
+              :handler #(re-frame/dispatch [:process-delete]))
+    app-db))
+
+; -- SAVE FOR TESTING ASYNC DOM UPDATE --
+;(re-frame/register-handler
+;  :update-tea
+;  (fn [app-db [_ id new-name]]
+;    (.setTimeout js/window #(ajax/GET (str "/api/teas/" id "/update/" new-name)
+;                                      :error-handler (fn [response]
+;                                                       (println "UPDATE-TEA ERROR" response))
+;                                      :handler (fn [response] (println response))) 4000)
+;    app-db)
+
+(re-frame/register-handler
+  :update-tea
+  (fn [app-db [_ id new-name]]
+    (ajax/GET (str "/api/teas/" id "/update/" new-name)
+              :error-handler (fn [response]
+                               (println "UPDATE-TEA ERROR " response))
+              :handler (fn [response] (re-frame/dispatch [:load-teas])))
+    app-db))
+
+
+(re-frame/register-handler
+  :process-delete
+  (fn [app-db [_ response]]
+    (re-frame/dispatch [:load-teas])
+    app-db))
+
+(re-frame/register-handler
+  :initialize-db
+  (fn [_ _]
+    {:teas-list []}))
+
+(re-frame/register-handler
+  :replace-tea
+  (fn [app-state [_ replaced-by edit-tea-input]]
+    (re-frame/dispatch [:update-tea (:id replaced-by) edit-tea-input])
+    (assoc-in app-state [:teas-list] (replace {replaced-by
+                                               {:id (:id replaced-by)
+                                                :name edit-tea-input}}
+                                              (:teas-list app-state)))))
+
+(re-frame/register-sub
+  :teas
+  (fn [db]
+    (reaction (:teas-list @db))))
+
+(re-frame/register-sub
+  :new-tea
+  (fn [db]
+    (reaction (:new-tea @db))))
+
+(re-frame/register-handler
+  :tea-posted
+  (fn [app-state]
+    (re-frame/dispatch [:load-teas])))
+
+(re-frame/register-handler
+  :create-tea
+  (fn [app-state [_ tea-name]]
+    (ajax/POST "/api/newtea" {:params {:new-tea tea-name}
+                              :format :json
+                              :handler #(re-frame/dispatch [:tea-posted])
+                              :error-handler (fn [err] (println "New Tea Failed To Post" err))})
+    app-state))
+
+(defn value-event
+  [event]
+  (-> event .-target .-value))
+
 (defn nav-link [uri title page collapsed?]
   [:li {:class (when (= page (session/get :page)) "active")}
    [:a {:href uri
         :on-click #(reset! collapsed? true)}
     title]])
-

 (defn navbar []
   (let [collapsed? (atom true)]
@@ -60,112 +165,6 @@
       [:div.col-md-12
        [:div {:dangerouslySetInnerHTML
               {:__html (md->html docs)}}]]])])
-
-(re-frame/register-handler
-  :process-teas-response
-  (fn [app-db [_ response]]
-    (assoc-in app-db [:teas-list] response)))
-
-(re-frame/register-handler
-  :process-teas-bad-response
-  (fn [app-db [_ response]]
-    (println "ERROR: " response)
-    app-db))
-
-(re-frame/register-handler
-  :load-teas
-  (fn [app-db _]
-    (ajax/GET "/api/teas"
-              {:handler #(re-frame/dispatch [:process-teas-response %1])
-               :error-handler #(re-frame/dispatch [:process-teas-bad-response %1])
-               :response-format :json
-               :keywords? true})
-    app-db))
-
-(re-frame/register-handler
-  :delete-tea
-  (fn [db [_ tea-name]]
-    (ajax/GET (str "/api/teas/" tea-name "/delete")
-              :error-handler (fn [response]
-                               (println "DELETE-TEA ERROR" response))
-              :handler (fn [response]
-                         (re-frame/dispatch [:load-teas])))))
-
-(re-frame/register-handler
-  :delete-tea
-  (fn [app-db [_ id]]
-    (ajax/GET (str "/api/teas/" id "/delete")
-              :error-handler (fn [response] (println "DELETE-TEA ERROR " response))
-              :handler #(re-frame/dispatch [:process-delete]))
-    app-db))
-
-; -- SAVE FOR TESTING ASYNC DOM UPDATE --
-;(re-frame/register-handler
-;  :update-tea
-;  (fn [app-db [_ id new-name]]
-;    (.setTimeout js/window #(ajax/GET (str "/api/teas/" id "/update/" new-name)
-;                                      :error-handler (fn [response]
-;                                                       (println "UPDATE-TEA ERROR" response))
-;                                      :handler (fn [response] (println response))) 4000)
-;    app-db)
-
-(re-frame/register-handler
-  :update-tea
-  (fn [app-db [_ id new-name]]
-    (ajax/GET (str "/api/teas/" id "/update/" new-name)
-              :error-handler (fn [response]
-                               (println "UPDATE-TEA ERROR " response))
-              :handler (fn [response] (re-frame/dispatch [:load-teas])))
-    app-db))
-
-
-(re-frame/register-handler
-  :process-delete
-  (fn [app-db [_ response]]
-    (re-frame/dispatch [:load-teas])
-    app-db))
-
-(re-frame/register-handler
-  :initialize-db
-  (fn [_ _]
-    {:teas-list []}))
-
-(re-frame/register-handler
-  :replace-tea
-  (fn [app-state [_ replaced-by edit-tea-input]]
-    (re-frame/dispatch [:update-tea (:id replaced-by) edit-tea-input])
-    (assoc-in app-state [:teas-list] (replace {replaced-by
-                                               {:id (:id replaced-by)
-                                                :name edit-tea-input}}
-                                              (:teas-list app-state)))))
-
-(re-frame/register-sub
-  :teas
-  (fn [db]
-    (reaction (:teas-list @db))))
-
-(re-frame/register-sub
-  :new-tea
-  (fn [db]
-    (reaction (:new-tea @db))))
-
-(re-frame/register-handler
-  :tea-posted
-  (fn [app-state]
-    (re-frame/dispatch [:load-teas])))
-
-(re-frame/register-handler
-  :create-tea
-  (fn [app-state [_ tea-name]]
-    (ajax/POST "/api/newtea" {:params {:new-tea tea-name}
-                              :format :json
-                              :handler #(re-frame/dispatch [:tea-posted])
-                              :error-handler (fn [err] (println "New Tea Failed To Post" err))})
-    app-state))
-
-(defn value-event
-  [event]
-  (-> event .-target .-value))

 (defn tea-edit-toggler
   [tea]