(maint) Cleanup benchmarking tool clojure syntax

--- src/puppetlabs/puppetdb/cli/benchmark.clj
+++ src/puppetlabs/puppetdb/cli/benchmark.clj
@@ -36,7 +36,6 @@
    `update` message with the current wall-clock. Each agent decides
    independently whether or not to submit a catalog during that clock
    tick."
-  (:import (java.io File))
   (:require [clojure.tools.logging :as log]
             [puppetlabs.puppetdb.catalog.utils :as catutils]
             [puppetlabs.trapperkeeper.logging :as logutils]
@@ -65,21 +64,20 @@

 (defn error-if
   [pred msg x]
-  (if (pred x) (log/error msg) x))
+  (if (pred x)
+    (log/error msg)
+    x))

 (defn load-sample-data
   "Load all .json files contained in `dir`."
-  ([dir]
-   (load-sample-data dir false))
-  ([dir from-classpath?]
-   (let [target-files (if from-classpath?
-                        (remove #(.isDirectory %) (file-seq (io/file (io/resource dir))))
-                        (fs/glob (fs/file dir "*.json")))]
-     (->> target-files
-          (map try-load-file)
-          (remove nil?)
-          (vec)
-          (error-if empty? (format "Supplied directory %s contains no usable data!" dir))))))
+  [dir & [from-classpath?]]
+  (let [target-files (if from-classpath?
+                       (remove #(.isDirectory %) (file-seq (io/file (io/resource dir))))
+                       (fs/glob (fs/file dir "*.json")))]
+    (->> target-files
+         (map try-load-file)
+         (filterv (complement nil?))
+         (error-if empty? (format "Supplied directory %s contains no usable data!" dir)))))

 (def mutate-fns
   "Functions that randomly change a wire-catalog format"
@@ -88,44 +86,49 @@
    catutils/add-random-edge-to-wire-catalog
    catutils/swap-edge-targets-in-wire-catalog])

-(defn rand-catalog-mutate-fn
+(defn rand-catalog-mutation
   "Grabs one of the mutate-fns randomly and returns it"
-  []
-  (rand-nth mutate-fns))
+  [catalog]
+  ((rand-nth mutate-fns) catalog))
+
+(declare random-fact-value-vector)

 (defn random-fact-value
   "Given a type, generate a random fact value"
-  [kind]
-  (case kind
-    :int (rand-int 300)
-    :float (rand)
-    :bool (random-bool)
-    :string (random-string 4)
-    :vector (into [] (take (rand-int 10)
-                           (repeatedly #(random-fact-value
-                                         (rand-nth [:string :int :float :bool])))))))
+  ([] (random-fact-value (rand-nth [:int :float :bool :string :vector])))
+  ([kind]
+   (case kind
+     :int (rand-int 300)
+     :float (rand)
+     :bool (random-bool)
+     :string (random-string 4)
+     :vector (random-fact-value-vector))))
+
+(defn random-fact-value-vector
+  []
+  (-> (rand-int 10)
+      (repeatedly #(random-fact-value (rand-nth [:string :int :float :bool])))
+      vec))

 (defn random-structured-fact
   "Create a 'random' structured fact.
   Parameters are fact depth and number of child facts.  Depth 0 implies one child."
   ([]
-     (random-structured-fact (rand-nth [0 1 2 3]) (rand-nth [1 2 3 4])))
+   (random-structured-fact (rand-nth [0 1 2 3])
+                           (rand-nth [1 2 3 4])))
   ([depth children]
-     (let [kind (rand-nth [:int :float :bool :string :vector])]
-       (if (zero? depth)
-         {(random-string 10) (random-fact-value kind)}
-         {(random-string 10) (zipmap (take children (repeatedly #(random-string 10)))
-                                     (take children (repeatedly
-                                                     #(random-structured-fact
-                                                       (rand-nth (range depth))
-                                                       (rand-nth (range children))))))}))))
+     (if (zero? depth)
+       {(random-string 10) (random-fact-value)}
+       {(random-string 10) (zipmap (repeatedly children #(random-string 10))
+                                   (repeatedly children #(random-structured-fact (rand-nth (range depth))
+                                                                                (rand-nth (range children)))))})))

 (defn maybe-tweak-catalog
   "Slightly tweak the given catalog, returning a new catalog, `rand-percentage`
   percent of the time."
   [catalog rand-percentage]
   (if (< (rand 100) rand-percentage)
-    ((rand-catalog-mutate-fn) catalog)
+    (rand-catalog-mutation catalog)
     catalog))

 (defn update-catalog [catalog]
@@ -137,19 +140,19 @@
    computing the same hash again (causing constraint errors in the DB)"
   [report]
   (assoc report
-    "configuration_version" (ks/uuid)
-    "start_time" (time/now)
-    "end_time" (time/now)
-    "producer_timestamp" (time/now)))
+         "configuration_version" (ks/uuid)
+         "start_time" (time/now)
+         "end_time" (time/now)
+         "producer_timestamp" (time/now)))

 (defn randomize-map-leaf
   "Randomizes a fact leaf based on a percentage provided with `rp`."
   [rp leaf]
   (if (< (rand 100) rp)
     (cond
-     (string? leaf)  (random-string (inc (rand-int 100)))
+     (string? leaf) (random-string (inc (rand-int 100)))
      (integer? leaf) (rand-int 100000)
-     (float? leaf)   (* (rand) (rand-int 100000))
+     (float? leaf) (* (rand) (rand-int 100000))
      (ks/boolean? leaf) (random-bool))
     leaf))

@@ -159,13 +162,10 @@
   [rp value]
   (cond
    (map? value)
-   (into {}
-         (for [[k v] value]
-           [k (randomize-map-leaves rp v)]))
+   (ks/mapvals (partial randomize-map-leaves rp) value)

    (coll? value)
-   (for [v value]
-     (randomize-map-leaves rp v))
+   (map (partial randomize-map-leaves rp) value)

    :else
    (randomize-map-leaf rp value)))
@@ -176,7 +176,7 @@
   [factset rand-percentage]
   (-> factset
       (assoc "producer_timestamp" (time/now))
-      (update-in ["values"] (partial randomize-map-leaves rand-percentage))))
+      (update "values" (partial randomize-map-leaves rand-percentage))))

 (defn timed-update-host
   "Send a new _clock tick_ to a host
@@ -191,11 +191,10 @@
     hosts catalog changes in accordance to the users preference)

   * Submit the resulting catalog"
-  [{:keys [host lastrun catalog report factset puppetdb-host puppetdb-port run-interval rand-percentage] :as state} clock]
+  [{:keys [host lastrun catalog report factset base-url run-interval rand-percentage] :as state} clock]
   (if-not (> (- clock lastrun) run-interval)
     state
-    (let [base-url {:protocol "http" :host puppetdb-host :port puppetdb-port :prefix "/pdb/query"}
-          catalog (some-> catalog update-catalog (maybe-tweak-catalog rand-percentage))
+    (let [catalog (some-> catalog update-catalog (maybe-tweak-catalog rand-percentage))
           report (some-> report update-report-run-fields)
           factset (some-> factset (update-factset rand-percentage))]
       ;; Submit the catalog and reports in separate threads, so as to not
@@ -230,9 +229,8 @@
    submission.  Also submit a report for the host (if present). This is
    similar to timed-update-host, but always sends the update (doesn't run/skip
    based on the clock)"
-  [{:keys [host lastrun catalog report factset puppetdb-host puppetdb-port run-interval rand-percentage] :as state}]
-  (let [base-url {:protocol "http" :host puppetdb-host :port puppetdb-port :prefix "/pdb/query"}
-        catalog (some-> catalog (maybe-tweak-catalog rand-percentage))
+  [{:keys [host lastrun catalog report factset base-url run-interval rand-percentage] :as state}]
+  (let [catalog (some-> catalog (maybe-tweak-catalog rand-percentage))
         report (some-> report update-report-run-fields)
         factset (some-> factset (update-factset rand-percentage))]
     (when catalog (client/submit-catalog base-url 6 (json/generate-string catalog)))
@@ -273,26 +271,6 @@
       (Thread/sleep 10)
       (recur curr-time))))

-(defn associate-catalog-with-host
-  "Takes the given `catalog` and transforms it to appear related to
-  `hostname`"
-  [hostname catalog]
-  (assoc catalog
-    "certname" hostname
-    "resources" (map #(update-in % ["tags"] conj hostname) (get catalog "resources"))))
-
-(defn associate-report-with-host
-  "Takes the given `report` and transforms it to appear related to
-  `hostname`"
-  [hostname report]
-  (assoc report "certname" hostname))
-
-(defn associate-factset-with-host
-  "Takes the given `factset` and transforms it to appear related to
-   `hostname`"
-  [hostname factset]
-  (assoc factset "certname" hostname))
-
 (def supported-cli-options
   [["-c" "--config CONFIG" "Path to config or conf.d directory (required)"]
    ["-F" "--facts FACTS" "Path to a directory containing sample JSON facts (files must end with .json)"]
@@ -310,7 +288,6 @@
 (defn activate-logging!
   [options]
   (-> (:config options)
-      config/load-config
       (get-in [:global :logging-config])
       logutils/configure-logging!))

@@ -321,8 +298,7 @@
     (and (contains? options :runinterval)
          (contains? options :nummsgs))
     (do
-      (log/error
-        "Error: -N/--nummsgs runs immediately and is not compatable with -i/--runinterval")
+      (log/error "Error: -N/--nummsgs runs immediately and is not compatable with -i/--runinterval")
       (action-on-error-fn))

     (not-any? #{:nummsgs :runinterval} (keys options))
@@ -333,25 +309,26 @@
     (and (contains? options :archive)
          (some #{:reports :catalogs :facts} (keys options)))
     (do
-      (log/error
-        "Error: -A/--archive is incompatible with -F/--facts, -C/--catalogs, -R/--reports")
+      (log/error "Error: -A/--archive is incompatible with -F/--facts, -C/--catalogs, -R/--reports")
       (action-on-error-fn))

     :else options))

 (defn default-options
   [options]
-  (-> options
-      (assoc :facts "puppetlabs/puppetdb/benchmark/samples/facts")
-      (assoc :reports "puppetlabs/puppetdb/benchmark/samples/reports")
-      (assoc :catalogs "puppetlabs/puppetdb/benchmark/samples/catalogs")
-      (assoc :from-cp? true)))
+  (assoc options
+         :facts "puppetlabs/puppetdb/benchmark/samples/facts"
+         :reports "puppetlabs/puppetdb/benchmark/samples/reports"
+         :catalogs "puppetlabs/puppetdb/benchmark/samples/catalogs"
+         :from-cp? true))

 (defn- validate-cli!
   [args]
   (try+
     (let [options (-> (ks/cli! args supported-cli-options required-cli-options)
                       first
+                      ;; We load the config for logging and for our main-
+                      (update :config config/load-config)
                       (validate-options #(System/exit 1)))]
       (if (empty? (select-keys options [:facts :reports :catalogs]))
         (default-options options)
@@ -370,67 +347,63 @@
         facts-pattern (re-pattern "facts.*\\.json$")]
     (fn [acc entry]
       (let [path (.getName entry)
-            parsed-entry (json/parse-string (archive/read-entry-content tar-reader))]
+            parsed-entry (-> tar-reader
+                             archive/read-entry-content
+                             json/parse-string)]
         (cond
-          (re-find catalog-pattern path) (update-in acc [:catalogs] conj parsed-entry)
-          (re-find report-pattern path) (update-in acc [:reports] conj parsed-entry)
-          (re-find facts-pattern path) (update-in acc [:facts] conj parsed-entry)
+          (re-find catalog-pattern path) (update acc :catalogs conj parsed-entry)
+          (re-find report-pattern path) (update acc :reports conj parsed-entry)
+          (re-find facts-pattern path) (update acc :facts conj parsed-entry)
           :else acc)))))

 (defn load-data-from-options
   [{:keys [catalogs facts reports archive from-cp?]}]
   (if archive
     (let [tar-reader (archive/tarball-reader archive)
-          entries (archive/all-entries tar-reader)]
-      (->> (reduce (process-tar-entry archive tar-reader)
-                   {:catalogs [] :reports [] :facts []} entries)
-           (filter (comp not empty? val))
+          process-entries-fn (process-tar-entry archive tar-reader)]
+      (->> (archive/all-entries tar-reader)
+           (reduce process-entries-fn {:catalogs [] :reports [] :facts []})
+           (remove (comp empty? val))
            (into {})))
-    {:catalogs (when catalogs
-                 (load-sample-data catalogs from-cp?))
-     :facts (when facts
-              (load-sample-data facts from-cp?))
-     :reports (when reports
-                (load-sample-data reports from-cp?))}))
+    {:catalogs (when catalogs (load-sample-data catalogs from-cp?))
+     :facts (when facts (load-sample-data facts from-cp?))
+     :reports (when reports (load-sample-data reports from-cp?))}))

 (defn -main
   [& args]
-  (let [options         (validate-cli! args)
-        config          (-> (:config options)
-                            config/load-config)
-
+  (let [{:keys [config] :as options} (validate-cli! args)
         {:keys [catalogs reports facts]} (load-data-from-options options)
-        nhosts          (:numhosts options)
-        hostnames       (set (map #(str "host-" %) (range (Integer/parseInt nhosts))))
-        hostname        (get-in config [:jetty :host] "localhost")
-        port            (get-in config [:jetty :port] 8080)
-        rand-percentage (Integer/parseInt (or (:rand-perc options) "0"))
-
+        {pdb-host :host pdb-port :port
+         :or {pdb-host "127.0.0.1" pdb-port 8080}} (:jetty config)
+        rand-percentage (Integer/parseInt (:rand-perc options "0"))
+        base-host-map {:base-url {:protocol "http"
+                                  :host pdb-host
+                                  :port pdb-port
+                                  :prefix "/pdb/query"}
+                       :rand-percentage rand-percentage}
         ;; Create an agent for each host
-        hosts (mapv (fn [host]
-                      {:host host
-                       :puppetdb-host hostname
-                       :puppetdb-port port
-                       :rand-percentage rand-percentage
-                       :catalog (when catalogs
-                                  (associate-catalog-with-host host (rand-nth catalogs)))
-                       :report (when reports
-                                 (associate-report-with-host host (rand-nth reports)))
-                       :factset (when facts
-                                   (associate-factset-with-host host (rand-nth facts)))})
-                    hostnames)]
-
-    (when-not catalogs
-      (log/info "No catalogs specified; skipping catalog submission"))
-    (when-not reports
-      (log/info "No reports specified; skipping report submission"))
-    (when-not facts
-      (log/info "No facts specified; skipping fact submission"))
+        nhosts (Integer/parseInt (:numhosts options))
+        hosts (->> (range nhosts)
+                   (map #(str "host-" %))
+                   (mapv (fn [host]
+                           (merge base-host-map
+                                  {:host host
+                                   :catalog (some-> catalogs
+                                                    rand-nth
+                                                    (assoc "certname" host)
+                                                    (update "resources" (partial map #(update % "tags" conj pdb-host))))
+                                   :report (some-> reports rand-nth (assoc "certname" host))
+                                   :factset (some-> facts rand-nth (assoc "certname" host))}))))]
+
+    (when-not catalogs (log/info "No catalogs specified; skipping catalog submission"))
+    (when-not reports (log/info "No reports specified; skipping report submission"))
+    (when-not facts (log/info "No facts specified; skipping fact submission"))
     (if-let [num-cmds (:nummsgs options)]
       (submit-n-messages hosts (Long/valueOf num-cmds))
-      (world-loop (mapv (fn [host-map]
-                          (let [run-interval (* 60 1000 (Integer/parseInt (:runinterval options)))]
-                            (agent (assoc host-map
-                                     :run-interval run-interval
-                                     :lastrun (- (System/currentTimeMillis) (rand-int run-interval))))))
-                        hosts)))))
+      (let [run-interval (* 60 1000 (Integer/parseInt (:runinterval options)))
+            rand-lastrun (fn [run-interval]
+                           (- (System/currentTimeMillis) (rand-int run-interval)))]
+        (->> hosts
+             (mapv #(agent (merge % {:run-interval run-interval
+                                     :lastrun (rand-lastrun run-interval)})))
+             world-loop)))))


(PDB-4512) handlers: move root-routes to be with the others

--- src/puppetlabs/puppetdb/http/handlers.clj
+++ src/puppetlabs/puppetdb/http/handlers.clj
@@ -51,6 +51,86 @@
    route-param-key :- s/Keyword]
   (cmdi/wrap-routes route #(parent-check % version entity route-param-key)))

+(defn report-data-responder
+  "Respond with either metrics or logs for a given report hash.
+   `entity` should be either :metrics or :logs."
+  [version entity]
+  (fn [{:keys [globals route-params]}]
+    (let [query ["from" entity ["=" "hash" (:hash route-params)]]]
+      (produce-streaming-body version {:query query}
+                              (http-q/narrow-globals globals)))))
+
+(defn route-param [param-name]
+  [#"[\w%\.~:-]*" param-name])
+
+;; Handlers checking for a single entity
+
+(defn status-response
+  "Executes `query` and if a result is found, calls `found-fn` with
+  that result, returns 404 otherwise."
+  [version query globals found-fn not-found-response]
+  (if-let [query-result (first (stream-query-result version query {} globals))]
+    (http/json-response (found-fn query-result))
+    not-found-response))
+
+(defn catalog-status
+  "Produces a response body for a request to retrieve the catalog for the node in route-params"
+  [version]
+  (fn [{:keys [globals route-params]}]
+    (let [node (:node route-params)]
+      (status-response version
+                       ["from" "catalogs" ["=" "certname" node]]
+                       (http-q/narrow-globals globals)
+                       #(s/validate catalog-query-schema
+                                    (kitchensink/mapvals sutils/parse-db-json [:edges :resources] %))
+                       (http/status-not-found-response "catalog" node)))))
+
+(defn factset-status
+  "Produces a response body for a request to retrieve the factset for the node in route-params"
+  [version]
+  (fn [{:keys [globals route-params]}]
+    (let [node (:node route-params)]
+      (status-response version
+                       ["from" "factsets" ["=" "certname" node]]
+                       (http-q/narrow-globals globals)
+                       identity
+                       (http/status-not-found-response "factset" node)))))
+
+(defn node-status
+  "Produce a response body for a single environment."
+  [version]
+  (fn [{:keys [globals route-params]}]
+    (let [node (:node route-params)]
+      (status-response version
+                       ["from" "nodes" ["=" "certname" node]]
+                       (http-q/narrow-globals globals)
+                       identity
+                       (http/status-not-found-response "node" node)))))
+
+(defn environment-status
+  "Produce a response body for a single environment."
+  [version]
+  (fn [{:keys [globals route-params]}]
+    (let [environment (:environment route-params)]
+      (status-response version
+                       ["from" "environments" ["=" "name" environment]]
+                       (http-q/narrow-globals globals)
+                       identity
+                       (http/status-not-found-response "environment" environment)))))
+
+(defn producer-status
+  "Produce a response body for a single producer."
+  [version]
+  (fn [{:keys [globals route-params]}]
+    (let [producer (:producer route-params)]
+      (status-response version
+                       ["from" "producers" ["=" "name" producer]]
+                       (http-q/narrow-globals globals)
+                       identity
+                       (http/status-not-found-response "producer" producer)))))
+
+;; Routes
+
 (pls/defn-validated root-routes :- bidi-schema/RoutePair
   [version :- s/Keyword]
   (let [no-certname-entities #{"fact_paths" "environments" "packages"}
@@ -64,86 +144,6 @@
                             :else (http-q/restrict-query-to-active-nodes req))))
                   (http-q/extract-query-pql {:optional (conj paging/query-params "ast_only")
                                              :required ["query"]})))))
-
-(defn report-data-responder
-  "Respond with either metrics or logs for a given report hash.
-   `entity` should be either :metrics or :logs."
-  [version entity]
-  (fn [{:keys [globals route-params]}]
-    (let [query ["from" entity ["=" "hash" (:hash route-params)]]]
-      (produce-streaming-body version {:query query}
-                              (http-q/narrow-globals globals)))))
-
-(defn route-param [param-name]
-  [#"[\w%\.~:-]*" param-name])
-
-;; Handlers checking for a single entity
-
-(defn status-response
-  "Executes `query` and if a result is found, calls `found-fn` with
-  that result, returns 404 otherwise."
-  [version query globals found-fn not-found-response]
-  (if-let [query-result (first (stream-query-result version query {} globals))]
-    (http/json-response (found-fn query-result))
-    not-found-response))
-
-(defn catalog-status
-  "Produces a response body for a request to retrieve the catalog for the node in route-params"
-  [version]
-  (fn [{:keys [globals route-params]}]
-    (let [node (:node route-params)]
-      (status-response version
-                       ["from" "catalogs" ["=" "certname" node]]
-                       (http-q/narrow-globals globals)
-                       #(s/validate catalog-query-schema
-                                    (kitchensink/mapvals sutils/parse-db-json [:edges :resources] %))
-                       (http/status-not-found-response "catalog" node)))))
-
-(defn factset-status
-  "Produces a response body for a request to retrieve the factset for the node in route-params"
-  [version]
-  (fn [{:keys [globals route-params]}]
-    (let [node (:node route-params)]
-      (status-response version
-                       ["from" "factsets" ["=" "certname" node]]
-                       (http-q/narrow-globals globals)
-                       identity
-                       (http/status-not-found-response "factset" node)))))
-
-(defn node-status
-  "Produce a response body for a single environment."
-  [version]
-  (fn [{:keys [globals route-params]}]
-    (let [node (:node route-params)]
-      (status-response version
-                       ["from" "nodes" ["=" "certname" node]]
-                       (http-q/narrow-globals globals)
-                       identity
-                       (http/status-not-found-response "node" node)))))
-
-(defn environment-status
-  "Produce a response body for a single environment."
-  [version]
-  (fn [{:keys [globals route-params]}]
-    (let [environment (:environment route-params)]
-      (status-response version
-                       ["from" "environments" ["=" "name" environment]]
-                       (http-q/narrow-globals globals)
-                       identity
-                       (http/status-not-found-response "environment" environment)))))
-
-(defn producer-status
-  "Produce a response body for a single producer."
-  [version]
-  (fn [{:keys [globals route-params]}]
-    (let [producer (:producer route-params)]
-      (status-response version
-                       ["from" "producers" ["=" "name" producer]]
-                       (http-q/narrow-globals globals)
-                       identity
-                       (http/status-not-found-response "producer" producer)))))
-
-;; Routes

 (pls/defn-validated events-routes :- bidi-schema/RoutePair
   "Ring app for querying events"


Added a var to tell the server the client state

In the realtime-clients global variable we add the item :update-request
so that when ever a web browser clients requests it’s realtime updates
we can just read the value of  :update-request and send it as the
response to the client

--- src/webapp/framework/server/systemfns.clj
+++ src/webapp/framework/server/systemfns.clj
@@ -386,6 +386,11 @@

+
+
+
+
+
 (defn make-log-table-update-trigger-function []
     (let [coils-trigger-fn-exists-result      (korma.core/exec-raw
                                                   ["select exists(select * from pg_proc where proname = 'trigger_function_afterUpdate');" []]
@@ -419,6 +424,10 @@

+
+
+
+
 (defn make-log-table-delete-trigger-function []
     (let [coils-trigger-fn-exists-result      (korma.core/exec-raw
                                                   ["select exists(select * from pg_proc where proname = 'trigger_function_afterDelete');" []]
@@ -449,6 +458,13 @@
             )
         )
     )
+
+
+
+
+
+
+

@@ -516,11 +532,16 @@

-;(sql-1 "select count(id) from table_name " [])
+

 (defn next-realtime-id [] (swap! realtime-counter inc))
+
+
+
+
+

@@ -562,6 +583,11 @@

+; ----------------------------------------------------------------
+; When the client application running in the web browser needs to be
+; updated in real time then the server needs to know which
+; SQL queries belong to which client so that it can update them
+; ----------------------------------------------------------------
 (defn add-client-to-query [query  client-id]
   (let [
         existing-query     (get @cached-queries   query)
@@ -571,9 +597,13 @@
         this-query         (get @client-queries query)
         ]
     (do
+      ; if the query does not know about the client then tell it
       (swap! existing-clients conj client-id)
+
+      ; if the client does not know about this query then tell it
       (if (not this-query)
         (swap! client-queries   assoc  query {}))
+
       (println "")
       (println "  add-client-to-query")
       (println (str "    " @realtime-clients))
@@ -648,7 +678,12 @@
   (let [client-cache     (get @realtime-clients  client-realtime-id)
         ]
     (if (nil? client-cache)
-      (swap! realtime-clients assoc client-realtime-id (atom {:queries (atom {})    :records (atom {})})))))
+      (swap! realtime-clients assoc client-realtime-id (atom {:queries (atom {})    :records (atom {})    :update-request (atom {})})))))
+
+
+
+
+

@@ -862,5 +897,8 @@
                                      client-data-session-id
                                      ]}]
   (do
+    (println (str "      " ))
     (println "SERVER: check-for-server-updates for client: " client-data-session-id)
+    (println (str "      " (keys @realtime-clients)))
+    (println (str "      " @(get @realtime-clients client-data-session-id)))
     []))


telnet to repl? much more responsive than spacemacs...

--- clojure/project.clj
+++ clojure/project.clj
@@ -8,6 +8,8 @@
                  [org.clojure/clojurescript "1.9.946"]
                  [com.rpl/specter "1.0.3"]
                  ]
+
+  ;:jvm-opts ["-Dclojure.server.repl={:port 5555 :accept clojure.core.server/repl}"]

   :plugins [[lein-cljsbuild "1.1.1"]
             [lein-figwheel "0.5.0-2"]]


Introduced abstraction for InputMap and ActionMap

for the repl process PlainDocument.

Signed-off-by: Daniel Borchmann <25857343a15bf1edafebc2912e82ecf775c585c1@mailbox.tu-dresden.de>

--- src/conexp/contrib/gui/repl.clj
+++ src/conexp/contrib/gui/repl.clj
@@ -173,15 +173,31 @@
 				      (insert-result result))))))]
     [repl-container repl-thread output-thread]))

+;;;
+
+(defn- add-input-event
+  "Adds a given input-event for the given key-sequence to
+  component. key-sequence must be a string describing a valid key
+  sequence and input-event must be a string."
+  [component key-sequence input-event]
+  (.. component getInputMap (put (KeyStroke/getKeyStroke key-sequence) input-event)))
+
+(defn- add-action-event
+  "Adds to component for a given input-event the callback to be called
+  when input-event is triggered. input must be a string describing an
+  input event and callback must be a function of no arguments."
+  [component input-event callback]
+  (.. component getActionMap (put input-event (proxy [AbstractAction] []
+                                                (actionPerformed [_]
+                                                  (callback))))))
+
 (defn- into-text-area
   "Puts repl-container (a PlainDocument) into a JTextArea adding some hotkeys."
   [repl-container repl-thread]
   (let [#^JTextArea repl-window (JTextArea. repl-container)]
-    (.. repl-window getInputMap (put (KeyStroke/getKeyStroke "control C") "interrupt"))
-    (.. repl-window getActionMap (put "interrupt" (proxy [AbstractAction] []
-						    (actionPerformed [_]
-						      (repl-interrupt repl-thread)))))
-    repl-window))
+    (doto repl-window
+      (add-input-event "control C" "interrupt")
+      (add-action-event "interrupt" #(repl-interrupt repl-thread)))))

 ;;;


Refactor: using existing fn

Refactor: using existing fn

--- src/cylon/oauth/impl/client.clj
+++ src/cylon/oauth/impl/client.clj
@@ -162,7 +162,7 @@

   AccessTokenGrantee
   (get-access-token [this req]
-    (when-let [app-session-id (-> req cookies-request :cookies (get APP-SESSION-ID) :value)]
+    (when-let [app-session-id (get-session-id req APP-SESSION-ID)]
       (-> (get-session (:session-store this) app-session-id))))

   (solicit-access-token [this req]
@@ -200,7 +200,7 @@

   UserIdentity
   (get-claims [this req]
-    (let [app-session-id (-> req cookies-request :cookies (get APP-SESSION-ID) :value)]
+    (let [app-session-id (get-session-id req APP-SESSION-ID)]
       (-> (get-session (:session-store this) app-session-id) :open-id)))

   ;; TODO Deprecate this!


[#200] Refactor nested `(if)` to `(cond)`

--- api/src/clojure/org/akvo/flow_api/endpoint/sync.clj
+++ api/src/clojure/org/akvo/flow_api/endpoint/sync.clj
@@ -57,21 +57,22 @@
                    :next-sync-url (next-sync-url api-root alias cursor)})
         (header "Cache-Control" "no-cache"))))

+(defn get-changes
+  [req cursor instance-id remote-api]
+  (let [db (:unilog-db-connection req)
+        offset (Long/parseLong cursor)]
+    (cond
+      (not (unilog/valid-offset? offset db)) (anomaly/bad-request "Invalid cursor" {})
+      (= offset (unilog/get-cursor db)) no-more-changes
+      :else (changes-response offset db instance-id remote-api req))))
+
 (defn changes [deps {:keys [alias instance-id params] :as req}]
   (let [{:keys [initial cursor next]} (spec/validate-params params-spec params)]
-    (if (and initial (or cursor next))
-      (anomaly/bad-request "Invalid parameters" {})
-      (if (= "true" initial)
-        (initial-response req)
-        (if (and next cursor)
-          (let [db (:unilog-db-connection req)
-                offset (Long/parseLong cursor)]
-            (if (unilog/valid-offset? offset db)
-              (if (= offset (unilog/get-cursor db)) ;; end of the log
-                no-more-changes
-                (changes-response offset db instance-id (:remote-api deps) req))
-              (anomaly/bad-request "Invalid cursor" {})))
-          (anomaly/bad-request "Invalid parameters" {}))))))
+    (cond
+      (and initial (or cursor next)) (anomaly/bad-request "Invalid parameters" {})
+      (= "true" initial) (initial-response req)
+      (and next cursor) (get-changes req cursor instance-id (:remote-api deps))
+      :else (anomaly/bad-request "Invalid parameters" {}))))

 (defn endpoint* [deps]
   (GET "/sync" req


Deal with the lexicon before the enterprise macro.

--- test/kolmogorov_music/test/kolmogorov.clj
+++ test/kolmogorov_music/test/kolmogorov.clj
@@ -20,34 +20,6 @@

 (fact "Sexprs can also be analysed for complexity."
   (kolmogorov/complexity (+ foo (88 "bar" true))) => 7)
-
-(defn minimal-complexity
-  "A hypothetical function that determines the minimal Kolmogorov complexity of a natural number."
-  [n]
-  (inc n))
-
-(defn first-that [applies? xs]
-  (->> xs
-       (drop-while (complement applies?))
-       first))
-
-(defn more-complex-than? [n limit]
-  (< limit (minimal-complexity n)))
-
-(defn enterprise*
-  "Find the first natural number with a complexity greater than f."
-  [expr ns]
-  (->> (range)
-       (first-that #(more-complex-than? % (kolmogorov/complexity* expr ns)))))
-
-(defmacro enterprise
-  [expr]
-  (enterprise* expr *ns*))
-
-(fact "The enterprise makes everything more complicated."
-  (enterprise inc) => 0
-  (enterprise baz) => 7
-  (enterprise enterprise) => 25)

 (def ascii
   (->> (range 32 127)
@@ -79,3 +51,31 @@
   (->> (lexicon) (subsequence 0 5)) => ["" " " "!" "\"" "#"]
   (->> (lexicon) (subsequence 95 100)) => ["~" "  " " !" " \"" " #"]
   (nth (lexicon) 364645) => "GEB")
+
+(defn minimal-complexity
+  "A hypothetical function that determines the minimal Kolmogorov complexity of a natural number."
+  [n]
+  (inc n))
+
+(defn first-that [applies? xs]
+  (->> xs
+       (drop-while (complement applies?))
+       first))
+
+(defn more-complex-than? [n limit]
+  (< limit (minimal-complexity n)))
+
+(defn enterprise*
+  "Find the first natural number with a complexity greater than f."
+  [expr ns]
+  (->> (range)
+       (first-that #(more-complex-than? % (kolmogorov/complexity* expr ns)))))
+
+(defmacro enterprise
+  [expr]
+  (enterprise* expr *ns*))
+
+(fact "The enterprise makes everything more complicated."
+  (enterprise inc) => 0
+  (enterprise baz) => 7
+  (enterprise enterprise) => 25)


cleanup

cleanup

--- ccw.core/src/ccw/debug/serverrepl.clj
+++ ccw.core/src/ccw/debug/serverrepl.clj
@@ -10,9 +10,7 @@
 ; *******************************************************************************/
 ; Totally footprint free embedded evaluation server
 ; (do not have any namespace / symbol presence)
-(ns ccw.debug.serverrepl
-  (:require [clojure.contrib.duck-streams :as cocks])
-  (:use clojure.contrib.repl-utils))
+(ns ccw.debug.serverrepl)

 ; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ; library code


add step!

add step!

--- test/osmium/core_test.clj
+++ test/osmium/core_test.clj
@@ -110,6 +110,7 @@
   (let [{:keys [actions step]} @replay
         next-action (nth actions step)]
     (eval! @replay-driver next-action)
-    (if (< (count actions) step)
-      (swap! step inc)
-      (reset! step 0))))
+    (if (< step (dec (count actions)))
+      (swap! replay update :step inc)
+      (swap! replay assoc :step 0))
+    next-action))


redefining environ/env here caused the jvm to exit

Specifically, I forgot to add in the :dev key so that runbld knew not
to exit.  I fixed the root cause (just take whatever the env has and
add to it) as well as added a fixture for the future.

--- test/runbld/main_test.clj
+++ test/runbld/main_test.clj
@@ -44,7 +44,13 @@

 (use-fixtures :each
   ts/redirect-logging-fixture
-  ts/reset-debug-log-fixture)
+  ts/reset-debug-log-fixture
+  (fn jvm-death-rattle [f]
+    (with-redefs [main/really-die (fn [& args]
+                                    ;; tattle if someone tries to kill the JVM
+                                    (println "SOMEBODY TRIED TO KILL THE JVM!")
+                                    :dontdie)]
+      (f))))

 (s/deftest main
   ;; Change root bindings for these Vars, affects any execution no
@@ -628,10 +634,6 @@
         intake-dir "tmp/git/metadata-intake"
         periodic-dir "tmp/git/metadata-periodic"]
     (try
-      (when (.exists (jio/file periodic-dir))
-        (io/rmdir-r periodic-dir))
-      (when (.exists (jio/file intake-dir))
-        (io/rmdir-r intake-dir))
       (with-redefs [email/send* (fn [_ _ _ _ _ plain html attachments]
                                   (reset! email-body html)
                                   ;; to satisfy schema
@@ -667,7 +669,9 @@
               (str "the metadata should be in the environment\n"
                    "the test is in check-metadata.bash"))
           (testing "existing BUILD_METADATA environment is respected"
-            (with-redefs [environ/env {:build-metadata "existing;metadata"}]
+            (with-redefs [environ/env (assoc environ/env
+                                             :build-metadata
+                                             "existing;metadata")]
               (let [[opts-periodic res-periodic]
                     (run (conj
                           ["-c" "test/config/main.yml"
@@ -677,7 +681,6 @@
                           "test/check-existing-metadata.bash"))]
                 ;; 0 exit code means the second script got the
                 ;; metadata- the check is in the bash script
-                (println (environ/env :build-metadata))
                 (is (= 0 (:exit-code res-periodic))
                     (str "the metadata was incorrect in the environment\n"
                          "the test is in check-existing-metadata.bash")))))))


updated date tests

updated date tests

--- test/selmer/core_test.clj
+++ test/selmer/core_test.clj
@@ -690,14 +690,14 @@
            (render "{{f|date:\"yyyy-MM-dd HH:mm:ss\"}}" {:f date})))
     (is (= (.format (java.text.SimpleDateFormat. "MMMM" (java.util.Locale. "fr")) firstofmarch)
            (render "{{f|date:\"MMMM\":fr}}" {:f firstofmarch})))
-    (is (= "12:00 AM" (render "{{d|date:shortTime:en_US}}" {:d firstofmarch})))
+    (is (= "00:00" (render "{{d|date:shortTime:en_US}}" {:d firstofmarch})))
     (is (= "上午12:00" (render "{{d|date:shortTime:zh}}" {:d firstofmarch})))
-    (is (= "3/1/14" (render "{{d|date:shortDate:en_US}}" {:d firstofmarch})))
-    (is (= "14-3-1" (render "{{d|date:shortDate:zh}}" {:d firstofmarch})))
-    (is (= "3/1/14 12:00 AM" (render "{{d|date:shortDateTime:en_US}}" {:d firstofmarch})))
-    (is (= "14-3-1 上午12:00" (render "{{d|date:shortDateTime:zh}}" {:d firstofmarch})))
+    (is (= "2014-03-01" (render "{{d|date:shortDate:en_US}}" {:d firstofmarch})))
+    (is (= "2014/3/1" (render "{{d|date:shortDate:zh}}" {:d firstofmarch})))
+    (is (= "2014-03-01 00:00" (render "{{d|date:shortDateTime:en_US}}" {:d firstofmarch})))
+    (is (= "2014/3/1 上午12:00" (render "{{d|date:shortDateTime:zh}}" {:d firstofmarch})))
     (is (= "2014年3月1日" (render "{{d|date:longDate:zh}}" {:d firstofmarch})))
-    (is (= "March 1, 2014" (render "{{d|date:longDate:en_US}}" {:d firstofmarch})))))
+    (is (= "2014 Mar 1" (render "{{d|date:longDate:en_US}}" {:d firstofmarch})))))

 (deftest filter-hash-md5
   (is (= "acbd18db4cc2f85cedef654fccc4a4d8"


Begin to refactor experiment helper.

Begin to refactor experiment helper.

--- src/dsbdp/experiment_helper.clj
+++ src/dsbdp/experiment_helper.clj
@@ -18,7 +18,7 @@
     (java.util HashMap Map)
     (org.apache.commons.math3.util CombinatoricsUtils)))

-(defmacro create-proc-fns
+(defn create-proc-fns
   [fn-1 fn-n n]
   (loop [fns (prewalk-replace {:idx 0} [fn-1])]
     (if (< (count fns) n)
@@ -29,28 +29,28 @@
         (println "proc-fns-pretty:\n" (.replaceAll (str (with-out-str (pprint fns))) "(?<=\\()([a-zA-Z\\.\\-]++/)" ""))
         fns))))

-(defmacro create-no-op-proc-fns
+(defn create-no-op-proc-fns
   [n]
-  `(create-proc-fns
-     (fn [~'_ ~'_])
-     (fn [~'_ ~'_])
-     ~n))
+  (create-proc-fns
+    (fn [_ _])
+    (fn [_ _])
+    n))

-(defmacro create-inc-proc-fns
+(defn create-inc-proc-fns
   [n]
-  `(create-proc-fns
-     (fn [~'i ~'_] (inc ~'i))
-     (fn [~'_ ~'o] (inc ~'o))
-     ~n))
+  (create-proc-fns
+    (fn [i _] (inc i))
+    (fn [_ o] (inc o))
+    n))

-(defmacro create-hashmap-inc-put-proc-fns
+(defn create-hashmap-inc-put-proc-fns
   [n]
   (let [o-sym 'o]
     (let [o-meta (vary-meta o-sym assoc :tag 'Map)]
-     `(create-proc-fns
-        (fn [~'i ~'_] (doto (HashMap.) (.put (str :idx) (inc ~'i))))
-        (fn [~'_ ~o-meta] (.put ~o-meta (str :idx) (inc (.get ~o-meta (str (dec :idx))))))
-        ~n))))
+     (create-proc-fns
+       (fn [i _] (doto (HashMap.) (.put (str :idx) (inc i))))
+       (fn [_ o-meta] (.put o-meta (str :idx) (inc (.get o-meta (str (dec :idx))))))
+       n))))

 (defn factorial
   [n]
@@ -59,10 +59,10 @@
       (recur (* result i) (inc i))
       result)))

-(defmacro create-factorial-proc-fns
+(defn create-factorial-proc-fns
   [n]
-  `(create-proc-fns
-     (fn [~'i ~'_] (factorial ~'i))
-     (fn [~'i ~'_] (factorial ~'i))
-     ~n))
+  (create-proc-fns
+    (fn [i _] (factorial i))
+    (fn [i _] (factorial i))
+     n))


fixed formatting

fixed formatting

--- src/ganelon/test/pages/ajax.clj
+++ src/ganelon/test/pages/ajax.clj
@@ -104,12 +104,12 @@
         [:p
          [:div.code
           [:pre.code
-           "(hiccup/include-js \"/ganelon/js/jquery-1.8.1.min.js\") ;jQuery - required
-           (hiccup/include-js \"/ganelon/js/bootstrap.js\") ;Bootstrap - optional
-           (hiccup/include-js \"/ganelon/js/ganelon.js\") ;basic actions support
-           (hiccup/include-js \"/ganelon/js/ext/ganelon.ops.bootstrap.js\") ;additional Bootstrap related actions
-           (hiccup/include-js \"/ganelon/js/ext/ganelon.ops.gritter.js\") ;growl-style notifications through gritter.js
-           (hiccup/include-js \"/ganelon/actions.js\") ;dynamic actions interface"
+"(hiccup/include-js \"/ganelon/js/jquery-1.8.1.min.js\") ;jQuery - required
+(hiccup/include-js \"/ganelon/js/bootstrap.js\") ;Bootstrap - optional
+(hiccup/include-js \"/ganelon/js/ganelon.js\") ;basic actions support
+(hiccup/include-js \"/ganelon/js/ext/ganelon.ops.bootstrap.js\") ;additional Bootstrap related actions
+(hiccup/include-js \"/ganelon/js/ext/ganelon.ops.gritter.js\") ;growl-style notifications through gritter.js
+(hiccup/include-js \"/ganelon/actions.js\") ;dynamic actions interface"
            ]]
          ]
         [:p "The Ganelon JavaScript files are available in " [:a {:href "https://github.com/tlipski/ganelon/tree/master/resources/public/ganelon/js"} "public.ganelon.js package"
@@ -119,7 +119,7 @@
         [:p
          [:div.code
           [:pre.code
-           "(hiccup/include-css \"/ganelon/css/bootstrap.css\") ;Bootstrap - optional
+"(hiccup/include-css \"/ganelon/css/bootstrap.css\") ;Bootstrap - optional
 (hiccup/include-css \"/ganelon/css/jquery.gritter.css\") ;growl-style notifications - optional"
            ]]
          ]


Add additional method test

Add additional method test

--- test/jest/method_test.clj
+++ test/jest/method_test.clj
@@ -43,3 +43,8 @@
    "y.foobar(2).bar(10).baz();"
    ["(baz (bar (foobar y 2) 10))"]))

+(deftest method-chain-test-5
+  (test-code
+   "y.foobar(2).bar(10, 20, 15).baz(true);"
+   ["(baz (bar (foobar y 2) 10 20 15) true)"]))
+


run start!/stop! as fixture

run start!/stop! as fixture

--- test/osmium/core_test.clj
+++ test/osmium/core_test.clj
@@ -122,9 +122,13 @@
   (and (or a b)
        (not (and a b))))

+(use-fixtures :each (fn [t]
+                      (o/start!)
+                      (t)
+                      (o/stop!)))
+
 (deftest users
   (testing "All created users are valid"
-    (o/start!)
     (let [driver (taxi/new-driver {:browser :firefox})
           db (:db @o/system)
           sessions (-> @o/system :session-store .session-map)]
@@ -140,8 +144,7 @@
                                          (is (xor (web/logged-in? session)
                                                   (login-button-rendered? source)))))))]
         (reset! replay {:actions actions :step 0}))
-      (taxi/quit driver))
-    (o/stop!)))
+      (taxi/quit driver))))

 (defn restart! []
   (swap! replay assoc :stop 0))


Hrefs now part of projects

Hrefs now part of projects

--- src/kixi/hecuba/web/project.clj
+++ src/kixi/hecuba/web/project.clj
@@ -2,10 +2,10 @@
   (:require
    [liberator.core :refer (defresource)]
    [kixi.hecuba.protocols :refer (upsert! item items)]
-   [bidi.bidi :refer (->Redirect)]
+   [bidi.bidi :refer (->Redirect path-for)]
+   [hiccup.core :refer (html)]
    [clojure.edn :as edn]
-   [clojure.java.io :as io]
-   ))
+   [clojure.java.io :as io]))

 (def base-media-types ["text/html" "application/json" "application/edn"])

@@ -14,10 +14,11 @@
   :available-media-types base-media-types
   :exists? (fn [{{{id :id} :route-params body :body routes :jig.bidi/routes} :request :as ctx}]
              {::projects
-              (map (fn [m] m #_(assoc m :href (path-for routes project-resource :id (:id m)))) (items querier))})
+              (map (fn [m] (assoc m :href (path-for routes project-resource :id (:id m)))) (items querier))})
   :handle-ok (fn [{projects ::projects {mt :media-type} :representation :as ctx}]
                (case mt
-                 "text/html" projects ; do something here to render the href as a link
+                 "text/html"
+                 (html [:table (for [p projects] [:tr [:td [:a {:href (:href p)} (:name p)]]])])
                  projects))
   :post! (fn [{{body :body} :request}]
            (let [payload (io! (edn/read (java.io.PushbackReader. (io/reader body))))]
@@ -28,9 +29,7 @@
   :available-media-types base-media-types
   :exists? (fn [{{{id :id} :route-params body :body routes :jig.bidi/routes} :request :as ctx}]
              (if-let [p (item querier id)] {::project p} false))
-  :handle-ok (fn [ctx] (::project ctx))
-  )
-
+  :handle-ok (fn [ctx] (::project ctx)))

 (defn create-routes [querier commander]
   (let [project (project-resource querier)


removing toggler UI for now

removing toggler UI for now

--- src/lt/plugins/lt_scrubber.cljs
+++ src/lt/plugins/lt_scrubber.cljs
@@ -2,10 +2,8 @@
   (:require [lt.object :as object]
             [lt.objs.command :as cmd]
             [lt.objs.editor :as editor]
-            [lt.objs.editor.pool :as pool]
-            [lt.util.dom :as dom]
-            [crate.binding :as binding])
-  (:require-macros [lt.macros :refer [defui behavior]]))
+            [lt.objs.editor.pool :as pool])
+  (:require-macros [lt.macros :refer [behavior]]))

 ;; These are some regex helper functions to find the position
 ;; of matches. This is currently not built into Clojurescript.
@@ -216,38 +214,19 @@
                         (.log js/console "eval on change")
                         (js/setTimeout #(cmd/exec! :eval-editor-form) 0))))

-;; This is a UI element that allows you to toggle the scrubbing
-;; functionality on or off
-(defui scrubber-toggler [this]
-  [:div#instarepl
-   [:span
-    {:class (binding/bound this #(str "livetoggler " (when-not (:active %) "off")))} "scrubber"
-    ]
-   ]
-  :click (fn [e]
-           (.log js/console "you clicked it!")
-           (dom/prevent e)
-           (object/raise (:editor @this) :scrubber.toggle!)))
-
-;; This Light Table object template contains the UI and whether
-;; the scrubber is active or not. It attaches the UI to the
-;; editor DOM
+;; This Light Table object template contains whether the
+;; scrubber is active or not.
 (object/object* ::scrubber
                 :tags #{::scrubber}
                 :name "Scrubber Toggler"
                 :active true
                 :init (fn [this editor]
-                        (object/merge! this {:editor editor})
-                        (let [editor-content (object/->content editor)
-                              frame (dom/parent editor-content)
-                              toggler (scrubber-toggler this)]
-                          (dom/append toggler editor-content)
-                          (dom/append frame toggler))))
-
-;; This function removes the scrubbing UI from the editor
+                        (object/merge! this {:editor editor})))
+
+;; This function removes the scrubbing behavior from the editor
 ;; and removes the mousedown event listener
 (defn scrubber-off [editor]
-  (print "scrubber-off")
+  (print "Scrubber Off")
   (object/remove-tags editor [:editor.scrubber.active])
   (when-let [scrubber (::scrubber @editor)]
     (object/merge! scrubber {:active false}))
@@ -259,7 +238,7 @@
 ;; exist, and adds it to the editor, as well as registering
 ;; event listeners for the mousedown event
 (defn scrubber-on [editor]
-  (print "scrubber-on")
+  (print "Scrubber On")
   (when-not (::scrubber @editor)
     (object/merge! editor {::scrubber (object/create ::scrubber editor)}))
   (object/add-tags editor [:editor.scrubber.active])
@@ -274,7 +253,6 @@
           :desc "Editor: Toggle scrubbing mode"
           :triggers #{:scrubber.toggle!}
           :reaction (fn [editor]
-                      (print "behavior scrubber-toggle")
                       (if (object/has-tag? editor :editor.scrubber.active)
                         (scrubber-off editor)
                         (scrubber-on editor))))


Default to diff for testing

Default to diff for testing

--- build.boot
+++ build.boot
@@ -23,7 +23,8 @@
                             :password (System/getenv "CLOJARS_PASS")}))]])

 (task-options! fmt
-               {:options {:fn-map {":require" :force-nl-body, "ns" :arg1-body},
+               {:mode :diff
+                :options {:fn-map {":require" :force-nl-body, "ns" :arg1-body},
                           :style :community,
                           :fn-force-nl #{:force-nl :noarg1 :noarg1-body
                                          :force-nl-body :binding}}})


remove audio notify for travis

remove audio notify for travis

--- build.boot
+++ build.boot
@@ -27,7 +27,7 @@
    You can use 'profile-tasks' like `production` and `development`
    to change parameters (like optimizations level of the cljs compiler)"
   []
-  (comp (speak)
+  (comp ;(speak)

         (cljs)
         ))


Added test for parsing string literals with escaped quotes

--- test/selmer/core_test.clj
+++ test/selmer/core_test.clj
@@ -675,6 +675,8 @@
   (testing "converts words to lower case"
     (is (= "foobar" (render "{{\"FOObar\"|lower}}" {})))
     (is (= "10" (render "{{1|add:2:3:4}}" {}))))
+  (testing "parses string literals with escaped quotes"
+    (is (= "foo\"bar" (render "{{\"foo\\\"BAR\"|lower}}" {}))))
   (testing "parses string literals with pipes"
     (is (= "foo|bar" (render "{{\"foo|BAR\"|lower}}" {})))))


Fix unused ns for Eastwood.

Fix unused ns for Eastwood.

--- src/oc/storage/api/activity.clj
+++ src/oc/storage/api/activity.clj
@@ -2,7 +2,6 @@
   "Liberator API for org resources."
   (:require [clojure.walk :refer (keywordize-keys)]
             [if-let.core :refer (if-let*)]
-            [taoensso.timbre :as timbre]
             [compojure.core :as compojure :refer (OPTIONS GET)]
             [liberator.core :refer (defresource by-method)]
             [oc.lib.slugify :as slugify]
@@ -13,7 +12,6 @@
             [oc.storage.api.access :as access]
             [oc.storage.representations.media-types :as mt]
             [oc.storage.representations.activity :as activity-rep]
-            [oc.storage.resources.common :as common-res]
             [oc.storage.resources.org :as org-res]
             [oc.storage.resources.board :as board-res]
             [oc.storage.resources.entry :as entry-res]))


Adds to information model.

Adds to information model.

--- src/onyx/information_model.cljc
+++ src/onyx/information_model.cljc
@@ -1931,7 +1931,7 @@
     :onyx/bulk?
     :onyx/restart-pred-fn]
    :flow-conditions-entry
-   [:flow/from :flow/to :flow/predicate :flow/exclude-keys :flow/short-circuit?
+   [:flow/from :flow/to :flow/predicate :flow/predicate-errors-to :flow/exclude-keys :flow/short-circuit?
     :flow/thrown-exception?  :flow/post-transform :flow/action :flow/doc]
    :window-entry
    [:window/id :window/task :window/type :window/aggregation :window/window-key


Fix information model.

Fix information model.

--- src/onyx/information_model.cljc
+++ src/onyx/information_model.cljc
@@ -670,6 +670,13 @@
              :type :integer
              :default 1000
              :added "0.8.3"}
+
+            :onyx.task-scheduler.colocated/only-send-local?
+            {:doc "When this peer is running a task for a job with a colocated task scheduler and this value is true, this peer will only send messages to segments local to its machine. It is desirable to set this to false when you want tasks to be perfectly uniformly spread over the machines in your cluster, but do not want jobs to run entirely locally."
+             :optional? true
+             :type :boolean
+             :default true
+             :added "0.8.4"}

             :onyx.messaging/inbound-buffer-size
             {:doc "Number of messages to buffer in the core.async channel for received segments."
@@ -1161,7 +1168,8 @@
     :onyx.rocksdb.filter/peer-block-cache-size
     :onyx.rocksdb.filter/num-buckets
     :onyx.rocksdb.filter/num-ids-per-bucket
-    :onyx.rocksdb.filter/rotation-check-interval-ms]
+    :onyx.rocksdb.filter/rotation-check-interval-ms
+    :onyx.task-scheduler.colocated/only-send-local?]
    :env-config
    [:onyx/id
     :zookeeper/server?


Fixed extraneous :as param in argspec

Fixed extraneous :as param in argspec

--- src/polismath/darwin/server.clj
+++ src/polismath/darwin/server.clj
@@ -232,7 +232,7 @@

 (defn filename-request-handler
   "Given aws-creds, returns a function handler function which responds to requests for an existing file on AWS."
-  [{:as darwin :keys [params postgres]} :as request]
+  [{:as darwin :keys [params postgres]} request]
   (let [{:keys [filename zinvite]} params
         zid (db/get-zid-from-zinvite postgres zinvite)]
     (if (= (get (get-export-status darwin filename zid) "status") "complete")
@@ -364,7 +364,7 @@
 (defn get-datadump-handler
   "Main handler function; Attempts to return a datadump file within within a set amount of time, and if it can't, will
   respond with a 202, and set up a process for obtaining the results once they're done."
-  [{:as darwin :keys [params postgres]} :as request]
+  [{:as darwin :keys [params postgres]} request]
   (let [params (parsed-params datadump-params params)
         ;; Check validity of params here
         _ (log/info "Handling datadump request with params:" (with-out-str (prn params)))


Remove active branches card

Remove active branches card

--- src-cljs/frontend/components/insights/project.cljs
+++ src-cljs/frontend/components/insights/project.cljs
@@ -106,10 +106,6 @@
                                   :formatter-use-start? true}})]]]
          [:div.card.insights-metadata
           [:dl
-           [:dt "active branches"]
-           [:dd (-> branches keys count)]]]
-         [:div.card.insights-metadata
-          [:dl
            [:dt "median queue"]
            [:dd (datetime/as-duration (insights/median (map :queued_time_millis bar-chart-builds)))]]]
          [:div.card.insights-metadata


Cleanup

Cleanup

--- src/reagent/impl/component.cljs
+++ src/reagent/impl/component.cljs
@@ -72,7 +72,7 @@
     :componentWillReceiveProps
     (fn [props]
       (this-as C
-               (when f (f C (aget props cljs-argv)))))
+               (f C (aget props cljs-argv))))

     :shouldComponentUpdate
     (fn [nextprops nextstate]
@@ -102,7 +102,8 @@
     (fn []
       (this-as C
                (batch/dispose C)
-               (when f (f C))))
+               (when-not (nil? f)
+                 (f C))))

     nil))


Make DB layer changes so that dynamo db schema mirrors API schema (quick hack).

--- src/rsvp_backend/db/invitee.clj
+++ src/rsvp_backend/db/invitee.clj
@@ -18,15 +18,15 @@
    client-opts
    db-name
    {:name name
-    :code code
-    :dateGiven date-given
-    :givenBy given-by
-    :preEnteredCompany pre-entered-company
-    :preEnteredTitle pre-entered-title
-    :originationSource origination-source
+    :code (Integer/parseInt code)
+    :date_given date-given
+    :given_by given-by
+    :pre_entered_company pre-entered-company
+    :pre_entered_title pre-entered-title
+    :origination_source origination-source
     :relationship relationship
     :confirmation "UNVERIFIED"
-    :rsvpState "UNVERIFIED"})
+    :rsvp_state "UNVERIFIED"})
   code)

@@ -34,41 +34,41 @@
   "Given the passcode, get the prefilled invitee details"
   [code]
   (ddb/update-item client-opts db-name {:code code}
-                   {:update-map {:rsvpState [:put "VERIFIED"]}})
+                   {:update-map {:rsvp_state [:put "VERIFIED"]}})
   (ddb/get-item client-opts db-name {:code code}))

 (defn update-invitee-metadata
   "Update prefilled invitee details"
-  [code {:keys [given-by origination-source]
-         :or {given-by nil origination nil}}]
+  [code {:keys [given_by origination_source]
+         :or {given_by nil origination-source nil}}]
   (ddb/update-item client-opts db-name {:code code}
-                   {:update-map {:givenBy [:put given-by]
-                                 :originationSource [:put origination-source]}}))
+                   {:update-map {:given_by [:put given_by]
+                                 :origination_source [:put origination_source]}}))

 (defn update-rsvp-status
   "Allow the user to respond with YES, NO or MAYBE"
   [code response]
   (ddb/update-item client-opts db-name {:code code}
                    {:update-map {:confirmation [:put response]
-                                 :rsvpState [:put "RESPONDED"]}}))
+                                 :rsvp_state [:put "RESPONDED"]}}))

 (defn update-invitee-details
   "Update the user's data with the submitted form data"
-  [code {:keys [name company title address address-2 city state zip
-                email mobile-phone work-phone website]
-         :or {name nil company nil title nil address nil address-2 nil city nil state nil zip
-              nil email nil mobile-phone nil work-phone nil website nil}}]
+  [code {:keys [name company title address address_2 city state zip
+                email mobile_phone work_phone website]
+         :or {name nil company nil title nil address nil address_2 nil city nil state nil zip
+              nil email nil mobile_phone nil work_phone nil website nil}}]
   (ddb/update-item client-opts db-name {:code code}
                    {:update-map {:name [:put name]
                                  :company [:put company]
                                  :title [:put title]
                                  :address [:put address]
-                                 :address2 [:put address-2]
+                                 :address_2 [:put address_2]
                                  :city [:put city]
                                  :state [:put state]
                                  :zip [:put zip]
                                  :email [:put email]
-                                 :mobilePhone [:put mobile-phone]
-                                 :workPhone [:put work-phone]
+                                 :mobile_phone [:put mobile_phone]
+                                 :work_phone [:put work_phone]
                                  :website [:put website]
-                                 :rsvpState [:put "FORM_SUBMITTED"]}}))
+                                 :rsvp_state [:put "FORM_SUBMITTED"]}}))


[Change] make-weights test

[Change] make-weights test

--- test/clj_elm/core_test.clj
+++ test/clj_elm/core_test.clj
@@ -24,9 +24,10 @@

 (facts "test-make-weights"
   (fact "(make-weights d)"
-    (map #(< -1.0 % 1.0)
-         (make-weights 100))
-    => (take 100 (repeat true))))
+    (reduce #(and %1 %2)
+            (map #(< -1.0 % 1.0)
+                 (make-weights 100)))
+    => true))

 (facts "test-make-ass"
   (fact "(make-ass d L)"


(maint) Bump pe-console-services dependency on tk-w-j9 to 0.7.7

--- configs/pe-console-services/pe-console-services.clj
+++ configs/pe-console-services/pe-console-services.clj
@@ -7,7 +7,7 @@
                  [puppetlabs/rbac-ui "{{{pe-rbac-ui-version}}}"]
                  [puppetlabs/pe-activity-service "{{{pe-activity-service-version}}}"]
                  [puppetlabs/pe-trapperkeeper-proxy "{{{pe-trapperkeeper-proxy-version}}}"]
-                 [puppetlabs/trapperkeeper-webserver-jetty9 "0.7.2"]]
+                 [puppetlabs/trapperkeeper-webserver-jetty9 "0.7.7"]]

   :uberjar-name "console-services-release.jar"


lose unused imports

lose unused imports

--- dada-demo/src/main/clojure/org/dada/demo/server.clj
+++ dada-demo/src/main/clojure/org/dada/demo/server.clj
@@ -12,14 +12,12 @@
   [org.springframework.beans.factory
    BeanFactory]
   [org.dada.core
-   ServiceFactory
    SessionManager
    SessionManagerNameGetter
    SessionManagerImpl
    View
    ViewNameGetter]
   [org.dada.core.jms
-   JMSServiceFactory
    POJOInvoker]
   [org.dada.jms
    SimpleMethodMapper]


Fixed merge conflict

Fixed merge conflict


Fix test failure

Fix test failure

--- test/re_frame/fx_test.cljs
+++ test/re_frame/fx_test.cljs
@@ -56,7 +56,7 @@
         log-fn           (fn [& args] (swap! logs conj (str/join args)))
         original-loggers (log/get-loggers)]
     (try
-      (log/set-loggers! {:error log-fn})
+      (log/set-loggers! {:warn log-fn})
       (re-frame/dispatch-sync [::missing-handler-test])
       (is (re-matches #"re-frame: no handler registered for effect::fx-not-exist. Ignoring." (first @logs)))
       (is (= (count @logs) 1))


fix sqs tests

fix sqs tests

--- test/shuppet/unit/core.clj
+++ test/shuppet/unit/core.clj
@@ -45,17 +45,17 @@
                    (shuppet/apply-config anything anything) => ..response.. :times 1))

             (fact "sqs message is sent when elb is created while applying a config"
-                  (apply-config ..env.. ..app..) => [{:action :CreateLoadBalancer
+                  (apply-config "poke" ..app..) => [{:action :CreateLoadBalancer
                                               :elb-name ..elb-name..}]
                   (provided
-                   (#'shuppet.core/*apply-config ..env.. ..app..) =>  [{:action :CreateLoadBalancer
+                   (#'shuppet.core/*apply-config "poke" ..app..) =>  [{:action :CreateLoadBalancer
                                                                 :elb-name ..elb-name..}]
-                   (sqs/announce-elb ..elb-name.. ..env.. ) => ..anything.. :times 1))
+                   (sqs/announce-elb ..elb-name.. "poke" ) => ..anything.. :times 1))

             (fact "sqs message is sent when elb is created while updating all configs"
-                  (update-configs ..env..) => [[{:action :CreateLoadBalancer
+                  (update-configs "poke") => [[{:action :CreateLoadBalancer
                                                  :elb-name ..elb-name..}]]
                   (provided
-                   (#'shuppet.core/concurrent-config-update ..env..) =>  [[{:action :CreateLoadBalancer
+                   (#'shuppet.core/concurrent-config-update "poke") =>  [[{:action :CreateLoadBalancer
                                                              :elb-name ..elb-name..}]]
-                   (sqs/announce-elb ..elb-name.. ..env.. ) => ..anything.. :times 1)))
+                   (sqs/announce-elb ..elb-name.. "poke" ) => ..anything.. :times 1)))


CMR-1308 Updates ISO MENDS to use new GML lib

--- umm-lib/src/cmr/umm/iso_mends/spatial.clj
+++ umm-lib/src/cmr/umm/iso_mends/spatial.clj
@@ -6,6 +6,7 @@
             [cmr.common.util :as util]
             [cmr.common.xml :as cx]
             [cmr.spatial.derived :as d]
+            [cmr.spatial.encoding.gml :as gml]
             [cmr.spatial.line-string :as ls]
             [cmr.spatial.mbr :as mbr]
             [cmr.spatial.point :as p]
@@ -23,52 +24,12 @@
   geographicElement elements. Dispatches on tag of element."
   :tag)

-(defmulti ^:private parse-gml
-  "Returns a UMM geometry value for a gml:* child of a
-  EX_BoundingPolygon element. Dispatches on tag."
-  :tag)
-
-;; EX_BoundingPolygon always contains a single gmd:polygon, which in
-;; turn contains a single gml:Point, gml:LineString, or gml:Polygon
-
 (defmethod parse-geo-element :EX_BoundingPolygon
   [element]
-  (let [gmd-polygon (-> element :content first)
-        gml-element (-> gmd-polygon :content first)]
-    (parse-gml gml-element)))
-
-(defmethod parse-gml :Point
-  [element]
-  (-> (cx/string-at-path element [:pos])
-      umm-s/lat-lon-point-str->points
-      first))
-
-(defmethod parse-gml :LineString
-  [element]
-  (-> (cx/string-at-path element [:posList])
-      umm-s/lat-lon-point-str->points
-      ls/line-string))
-
-(defmethod parse-gml :Polygon
-  [element]
-  (let [close-ring (fn [points]
-                     (let [v (vec points)]
-                       (if (= (first v) (last v))
-                         v
-                         (conj v (first v)))))
-        exterior  (-> (cx/string-at-path element [:exterior :LinearRing :posList])
-                      umm-s/lat-lon-point-str->points
-                      close-ring)
-        interiors (->> (cx/strings-at-path element [:interior :LinearRing :posList])
-                       (map umm-s/lat-lon-point-str->points)
-                       (map close-ring))
-        rings     (cons (umm-s/ring exterior)
-                        (map umm-s/ring interiors))]
-    (poly/polygon rings)))
-
-(defmethod parse-geo-element :default
-  [_]
-  nil)
+  ;; EX_BoundingPolygon elements always contain a single gmd:polygon,
+  ;; which in turn contain a single gml:Point, gml:LineString, or
+  ;; gml:Polygon
+  (gml/decode (-> element :content first :content first)))

 (defmethod parse-geo-element :EX_GeographicBoundingBox
   [element]
@@ -77,6 +38,10 @@
         north (cx/double-at-path element [:northBoundLatitude :Decimal])
         south (cx/double-at-path element [:southBoundLatitude :Decimal])]
     (mbr/mbr west north east south)))
+
+(defmethod parse-geo-element :default
+  [_]
+  nil)

 (defn- parse-geometries
   "Returns a seq of UMM geometry records from an ISO XML document."
@@ -114,75 +79,43 @@
   {:srsName "http://www.opengis.net/def/crs/EPSG/4326"
    :srsDimension "2"})

-(defn gml-pos
-  [point]
-  (x/element :gml:pos pos-attrs (pos-str point)))
-
-(defn gml-poslist
-  [points]
-  (x/element :gml:posList pos-attrs (apply pos-str points)))
-
-(defn gmd-poly
-  [content]
-  (x/element :gmd:EX_BoundingPolygon {}
-             (x/element :gmd:polygon {}
-                        content)))
-
-(defn gml-linear-ring
-  [points]
-  (x/element :gml:LinearRing {}
-             (gml-poslist points)))
-
 (defn gco-decimal
   "Returns n as a gco:Decimal element"
   [n]
   (x/element :gco:Decimal {} (util/double->string n)))

+;;; Generating ISO MENDS XML geographic extent elements
+
 (defmulti geometry->iso-geom
   "Returns content of a ISO MENDS extent element for an individual UMM
   spatial coverage geometry record obj. Dispatches on the geometry
-  record type. gen-id must be a function which returns a
-  document-level unique ID for a geographicElement."
-  (fn [obj gen-id] (type obj)))
-
-;;; Rendering gmd:geographicElement content
-
-(defmethod geometry->iso-geom cmr.spatial.point.Point
-  [point gen-id]
-  (gmd-poly (x/element :gml:Point {:gml:id (gen-id)} (gml-pos point))))
-
-(defmethod geometry->iso-geom cmr.spatial.line_string.LineString
-  [{:keys [points]} gen-id]
-  (gmd-poly (x/element :gml:LineString {:gml:id (gen-id)}
-                       (gml-poslist points))))
-
-(defmethod geometry->iso-geom cmr.spatial.polygon.Polygon
-  [polygon gen-id]
-  (let [exterior (:points (poly/boundary polygon))
-        interior (map :points (poly/holes polygon))]
-    (gmd-poly
-     (x/element :gml:Polygon {:gml:id (gen-id)}
-                (x/element :gml:exterior {} (gml-linear-ring exterior))
-                (when (seq interior)
-                  (map #(x/element :gml:interior {} (gml-linear-ring %)) interior))))))
+  record type. id must be a document-level unique ID for a
+  geographicElement."
+  (fn [obj id] (type obj)))

 (defmethod geometry->iso-geom cmr.spatial.mbr.Mbr
-  [mbr gen-id]
+  [mbr _]
   (x/element :gmd:EX_GeographicBoundingBox {}
              (x/element :gmd:westBoundLongitude {} (gco-decimal (:west mbr)))
              (x/element :gmd:eastBoundLongitude {} (gco-decimal (:east mbr)))
              (x/element :gmd:southBoundLatitude {} (gco-decimal (:south mbr)))
              (x/element :gmd:northBoundLatitude {} (gco-decimal (:north mbr)))))

+(defmethod geometry->iso-geom :default
+  [obj id]
+  (x/element :gmd:EX_BoundingPolygon {}
+             (x/element :gmd:polygon {}
+                        (assoc-in (gml/encode obj) [:attrs :gml:id] id))))
+
 (defn geometry->iso-xml
   "Returns an individual ISO MENDS geographic extent element for a UMM
   spatial coverage geometry record."
   [coordinate-system geom gen-id]
   ;; set the coordinate system based on the spatial coverage for output
-  (let [geom     (d/calculate-derived (umm-s/set-coordinate-system coordinate-system geom))]
+  (let [geom (d/calculate-derived (umm-s/set-coordinate-system coordinate-system geom))]
     (list
-     (x/element :gmd:geographicElement {} (geometry->iso-geom (r/mbr geom) gen-id))
-     (x/element :gmd:geographicElement {} (geometry->iso-geom geom gen-id)))))
+     (x/element :gmd:geographicElement {} (geometry->iso-geom (r/mbr geom) (gen-id)))
+     (x/element :gmd:geographicElement {} (geometry->iso-geom geom (gen-id))))))

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Public Functions
@@ -197,7 +130,7 @@
       (c/map->SpatialCoverage
        {:spatial-representation coord-sys
         :granule-spatial-representation coord-sys
-        :geometries geometries}))))
+        :geometries (map #(umm-s/set-coordinate-system coord-sys %) geometries)}))))

 (defn spatial-coverage->coordinate-system-xml
   "Returns ISO MENDS coordinate system XML element from the given SpatialCoverage."


changed lat lon format

changed lat lon format

--- extras/location-extraction/src/location.clj
+++ extras/location-extraction/src/location.clj
@@ -32,10 +32,9 @@

 ;;extract data from GeoName object
 (defn info [geoName]
-  (let [keys [:latlon :name :preferredName]
-        latlon (juxt #(. % getLatitude)
-                     #(. % getLongitude))
-        funs (juxt latlon
+  (let [keys [:location :name :preferredName]
+        funs (juxt (fn [o] {:lat (.getLatitude o)
+                           :lon (.getLongitude o)})
                    #(. % getName)
                    #(. % getPreferredName))]
     (zipmap keys (funs geoName))))


Add roundtrip test that warms up code

Add roundtrip test that warms up code

--- test/transit/roundtrip.clj
+++ test/transit/roundtrip.clj
@@ -96,7 +96,16 @@
   (let [res (rt-raw form)]
     (into {} (map (fn [[k v]] [k (dissoc v :form)]) res))))

+(defn rt-summary-warm
+  [n form]
+  (dotimes [i n]
+    (prn i)
+    (rt-summary form))
+  (rt-summary form))
+
 (comment

 )
+
+


Remove :ruler from default workspace flags.

Mistake introduced in previous commit.

--- frontend/src/uxbox/main/data/workspace.cljs
+++ frontend/src/uxbox/main/data/workspace.cljs
@@ -50,7 +50,7 @@
 (defrecord InitializeWorkspace [project page]
   ptk/UpdateEvent
   (update [_ state]
-    (let [default-flags #{:sitemap :drawtools :layers :element-options :rules :ruler}]
+    (let [default-flags #{:sitemap :drawtools :layers :element-options :rules}]
       (if (:workspace state)
         (update state :workspace merge
                 {:project project


Minor fix on colors page initialization event.

--- src/uxbox/main/data/colors.cljs
+++ src/uxbox/main/data/colors.cljs
@@ -38,17 +38,11 @@
 (defrecord Initialize [type id]
   rs/UpdateEvent
   (-apply-update [_ state]
-    (let [type (or type :builtin)
-          id (or id (if (= type :builtin) 1 nil))
+    (let [type (or type :own)
           data {:type type :id id :selected #{}}]
       (-> state
           (assoc-in [:dashboard :colors] data)
           (assoc-in [:dashboard :section] :dashboard/colors))))
-
-  ;; rs/EffectEvent
-  ;; (-apply-effect [_ state]
-  ;;   (when (nil? (:color-colls-by-id state))
-  ;;     (reset! st/loader true)))

   rs/WatchEvent
   (-apply-watch [_ state s]


Check the mime type higher in the call stack

Check the mime type before we attempt to fetch the thumb on CEPH and
before we download the original. This should save two potential round
trips in the case where we attempt to thumb an unsupported video file.

--- src/vignette/util/thumbnail.clj
+++ src/vignette/util/thumbnail.clj
@@ -37,7 +37,7 @@
 (defn assert-original-mime-type
   [file thumb-map]
   (let [mime-type (mime-type-of file)]
-    (when (contains? unsupported-mime-types (mime-type-of file))
+    (when (contains? unsupported-mime-types mime-type)
       (throw+ {:type :convert-error
                :response-code 404
                :content-type mime-type
@@ -58,7 +58,6 @@

 (defn original->thumbnail
   [resource thumb-map]
-  (assert-original-mime-type resource thumb-map)
   (let [temp-file (temp-filename (str (wikia thumb-map) "_thumb"))
         base-command [thumbnail-bin
                       "--in" (.getAbsolutePath resource)
@@ -79,6 +78,7 @@

 (defn get-or-generate-thumbnail
   [system thumb-map]
+  (assert-original-mime-type (get thumb-map :original) thumb-map)
   (if-let [thumb (and (not (q/query-opt thumb-map :replace))
                       (get-thumbnail (store system) thumb-map))]
     thumb


updates check for service-mapping exclusive being enabled (#1493)

--- waiter/src/waiter/util/client_tools.clj
+++ waiter/src/waiter/util/client_tools.clj
@@ -1322,6 +1322,11 @@
 (defn exclusive-mode-by-default?
   "Returns true if service-mapping is exclusive by default."
   [waiter-url]
-  (-> (waiter-settings waiter-url)
-    (get-in [:token-config :token-defaults :service-mapping])
-    (= "exclusive")))
+  (let [settings (waiter-settings waiter-url)
+        service-mapping (get-in settings [:token-config :token-defaults :service-mapping])]
+    (or (= service-mapping "exclusive")
+        (and (= service-mapping "default")
+             (-> settings
+               (get-in [:token-config :exclusive-promotion-start-time] "2050-06-01T00:00:00.000Z")
+               (du/str-to-date)
+               (t/before? (t/now)))))))


Now working on 0.11.0-SNAPSHOT

Now working on 0.11.0-SNAPSHOT

--- metrics-clojure-core/project.clj
+++ metrics-clojure-core/project.clj
@@ -1,4 +1,4 @@
-(defproject metrics-clojure "0.10.0"
+(defproject metrics-clojure "0.11.0-SNAPSHOT"
   :description "A Clojure façade for Coda Hale's metrics library."
   :dependencies [[org.clojure/clojure             "1.5.1"]
                  [com.yammer.metrics/metrics-core "2.2.0"]]


Increase number of search results.

Increase number of search results.

--- src/clj/freestuffly/notifier.clj
+++ src/clj/freestuffly/notifier.clj
@@ -17,7 +17,7 @@
       (re-find (re-matcher interesting-keywords (first (:content (into {} result))))))
     (set results)))

-(def my-group-urls "https://groups.freecycle.org/group/southwark-freecycle/posts/offer?page=1&resultsperpage=3&showall=off&include_offers=off&include_wanteds=off&include_receiveds=off&include_takens=off")
+(def my-group-urls "https://groups.freecycle.org/group/southwark-freecycle/posts/offer?page=1&resultsperpage=50&showall=off&include_offers=off&include_wanteds=off&include_receiveds=off&include_takens=off")

 (def site-tree
   (-> (client/get my-group-urls) :body h/parse h/as-hickory))


atom-sync: add ^:no-doc marks for private stuff

--- contrib/atom-sync/src/carry_atom_sync/core.cljs
+++ contrib/atom-sync/src/carry_atom_sync/core.cljs
@@ -1,7 +1,7 @@
 (ns carry-atom-sync.core
   (:require [cljs.core.match :refer-macros [match]]))

-(defn -connect-atom
+(defn ^:no-doc -connect-atom
   [r model dispatch-signal]
   ; sync model from reference
   (add-watch r
@@ -17,7 +17,7 @@
                (when (not= @r new-state)
                  (reset! r new-state)))))

-(defn -unconnect-atom
+(defn ^:no-doc -unconnect-atom
   [r model]
   (remove-watch r model)
   (remove-watch model r))


Query processor 'sum' now works

Query processor 'sum' now works

--- src/metabase/driver/postgres.clj
+++ src/metabase/driver/postgres.clj
@@ -10,7 +10,13 @@
 (declare build-query
          generate-sql)

-(defn annotate-column [table column]
+(defn annotate-column
+  "Add the following keys to COLUMN (used internally):
+  *  `:keyword` Used for keying some internal maps (?)
+  *  `:qualified-name` Name to be used in the `SELECT` statement, e.g. `\"main_guide\".\"user_id\"`
+  *  `:alias` Alias for the field in `SELECT` statements (needed in some cases where we cast fields such as Dates)
+  *  `:select` Combined qualified name/alias form to use in `SELECT` statement, e.g. `\"main_guide\".\"user_id\" AS `\"main_guide\"`"
+  [table column]
   (let [{:keys [name base_type]} column
         qualified-name (format "\"%s\".\"%s\"" (:name table) name)
         alias (match base_type
@@ -27,7 +33,10 @@
            :alias alias
            :select select)))

-(defn annotate-column-with-id [table column-id]
+(defn annotate-column-with-id
+  "Fetch `Field` with COLUMN-ID and call `annotate-column`."
+  [table column-id]
+  {:pre [(integer? column-id)]}
   (annotate-column table (sel :one Field :id column-id)))

 (defn annotate-special-column [column]
@@ -43,11 +52,31 @@
              :id nil
              :description nil}))

+(defn is-special-aggregation?
+  "Is this aggregation clause a 'special' case such as `sum`?"
+  [aggregation]
+  (match (first aggregation)
+    "rows" true
+    "sum" true
+    :else false))
+
+(defn apply-special-aggregation [{:keys [source_table table aggregation] :as query}]
+  (case (first aggregation)
+    "rows" (->> (sel :many Field :table_id source_table) ; aggregation: ["rows"] just means return all rows
+                (map (partial annotate-column table)))
+    "sum" (let [{:keys [name] :as field} (annotate-column-with-id table (second aggregation))]
+            (->> (assoc field
+                       :name (format "Sum (%s)" name)
+                       :keyword :sum
+                       :alias :sum
+                       :select (format "SUM(\"%s\".\"%s\")" (:name table) name))
+                 vector))))
+
 (defn get-columns
-  "Return an array of column info dictionaries for query."
+  "Return an array of column info dictionaries for QUERY."
   [{:keys [source_table table breakout aggregation] :as query}]
-  (if (= aggregation ["rows"]) (->> (sel :many Field :table_id source_table) ; aggregation: ["rows"] just means return all rows
-                                    (map (partial annotate-column table)))
+  {:post [sequential?]}
+  (if (is-special-aggregation? aggregation) (apply-special-aggregation query)
       (->> (concat breakout aggregation)
            (filter identity)
            (map (fn [column]
@@ -61,15 +90,15 @@
           :database (:db (-> (:source_table <>)
                              (hydrate :db)))
           :columns (get-columns <>)
-          :columns-by-name (->> (:columns <>)
-                                (map (fn [{:keys [keyword] :as col}]
-                                       {keyword col}))
-                                (reduce merge {}))
-          :columns-by-id (->> (:columns <>)
-                              (filter :id)
-                              (map (fn [{:keys [id] :as col}]
-                                     {id col}))
-                              (reduce merge {}))
+          :column-name->column (->> (:columns <>)
+                                    (map (fn [{:keys [keyword] :as col}]
+                                           {keyword col}))
+                                    (reduce merge {}))
+          :column-id->column (->> (:columns <>)
+                                  (filter :id)
+                                  (map (fn [{:keys [id] :as col}]
+                                         {id col}))
+                                  (reduce merge {}))
           :ordered-columns (map :keyword (:columns <>))
           :ordered-aliases (map #(keyword (:alias %)) (:columns <>))
           :sql (->> <>
@@ -78,7 +107,7 @@

 (defn apply-breakout [query fields]
   (let [field-names (->> fields
-                  (map (:columns-by-id query))
+                  (map (:column-id->column query))
                   (map :alias)
                   (interpose ", ")
                   (apply str))]
@@ -131,7 +160,7 @@
          (apply str))))

 (defn process-and-run [{:keys [database] :as query}]
-  (let [{:keys [sql columns ordered-columns columns-by-name] :as query-dict} (process (:query query))
+  (let [{:keys [sql columns ordered-columns column-name->column] :as query-dict} (process (:query query))
         db (sel :one Database :id database)
         results ((:native-query db) sql)]
     (println "----------------------------------------\n"
@@ -141,8 +170,8 @@
     {:status :completed
      :row_count (count results)
      :data {:rows (->> results
-                       (map #(map % (:ordered-aliases query-dict)))
-                       (map format-result))
-            :columns (map :name columns)
-            :cols (map columns-by-name
-                       ordered-columns)}}))
+                       (mapv #(mapv % (:ordered-aliases query-dict)))
+                       (mapv format-result))
+            :columns (mapv :name columns)
+            :cols (mapv column-name->column
+                        ordered-columns)}}))


fixed disconnect when not signed in

fixed disconnect when not signed in


(maint) Merge up 42c3a21 to main

(maint) Merge up 42c3a21 to main

Generated by CI

* commit '42c3a2160a034789712e9eece540f5f677878fc4':
  (maint) Test leaking threads by identity vs name
  (maint) update submodule versions and agent pin
  (maint) update puppet submodule version and agent pin

--- test/integration/puppetlabs/services/jruby/jruby_puppet_pool_int_test.clj
+++ test/integration/puppetlabs/services/jruby/jruby_puppet_pool_int_test.clj
@@ -629,10 +629,8 @@
        jruby-testutils/jruby-service-and-dependencies
        (jruby-testutils/jruby-puppet-tk-config
         (jruby-testutils/jruby-puppet-config {:max-active-instances 1}))
-        ;; getName vs getId here is helpful in debugging
-        (let [threads-before-jruby (->> (Thread/getAllStackTraces)
-                                        (map #(.getName (key %)))
-                                        (set))
+        ;; getAllStackTraces returns a HashMap of Thread to StackTraceElement[]
+        (let [threads-before-jruby (set (keys (Thread/getAllStackTraces)))
               jruby-service (tk-app/get-service app :JRubyPuppetService)
               pool-context (jruby-protocol/get-pool-context jruby-service)
               pool-agent (jruby-agents/get-modify-instance-agent pool-context)
@@ -644,17 +642,13 @@
           (try
             (.runScriptlet scripting-container script)
             (catch EvalFailedException e))
-          (let [threads-during-jruby (->> (Thread/getAllStackTraces)
-                                          (map #(.getName (key %)))
-                                          (set))]
+          (let [threads-during-jruby (set (keys (Thread/getAllStackTraces)))]
             (jruby-testutils/return-instance jruby-service instance :test)
             (jruby-protocol/flush-jruby-pool! jruby-service)
             ; wait until the flush is complete
             (await pool-agent)
             (Thread/sleep 5000)
-            (let [threads-after-jruby (->> (Thread/getAllStackTraces)
-                                           (map #(.getName (key %)))
-                                           (set))
+            (let [threads-after-jruby (set (keys (Thread/getAllStackTraces)))
                   threads-created-by-jruby (set/difference threads-during-jruby threads-before-jruby)
                   threads-orphaned-by-jruby (set/intersection threads-created-by-jruby threads-after-jruby)]
               (is (empty? threads-orphaned-by-jruby)))))))))


[scamp] Mark heartbeats (§2.2.3) DONE

[scamp] Mark heartbeats (§2.2.3) DONE

--- modules/scamp/src/scamp/core.clj
+++ modules/scamp/src/scamp/core.clj
@@ -10,7 +10,9 @@
     * DONE: subscription (§2.2.1, Algorithm 1)
     * DONE: subscription (§2.2.1, Algorithm 2)
     * DONE: unsubscription (§2.2.2)
-    * heartbeats (§2.2.3) (partially done, first async feature because depends on concept of \"a long time\")
+    * DONE: heartbeats (§2.2.3) (This is the first async feature,
+                                 because it depends on the concept of
+                                 \"a long time\".)
     * subscription indirection (§3.1)
     * lease mechanism (§3.2)
   "


Simplify without-logging-status

Simplify without-logging-status

--- test/clj/tapp/utils/logging.clj
+++ test/clj/tapp/utils/logging.clj
@@ -39,7 +39,7 @@
     #'mount.core/sigcont})

 (defn without-logging-status []
-  (doall (map #(clear-hooks %) lifecycle-fns)))
+  (doall (map clear-hooks lifecycle-fns)))

 (defn with-logging-status []
   (without-logging-status)


removed unneeded def statements

removed unneeded def statements

--- test/lcmap/clownfish/shared.clj
+++ test/lcmap/clownfish/shared.clj
@@ -9,9 +9,6 @@
             [lcmap.clownfish.setup.finalize :as finalize]
             [lcmap.clownfish.system :as system]
             [org.httpkit.client :as http]))
-
-(def http-port (get-in config [:http :port]))
-(def http-host (str "http://localhost:" http-port))

 (defmacro with-system
   "Start and stop the system, useful for integration tests."


Add test for multi-spec's 'no method' case

--- test/pinpointer/trace_test.cljc
+++ test/pinpointer/trace_test.cljc
@@ -2,6 +2,9 @@
   (:require [clojure.test :refer [deftest are]]
             [clojure.spec.alpha :as s]
             [pinpointer.trace :as trace]))
+
+(defmulti shape-spec :type)
+(s/def ::shape (s/multi-spec shape-spec :type))

 (deftest trace-test
   (are [spec input expected]
@@ -25,4 +28,11 @@
     [[[{:spec `(s/cat :first integer? :second integer?)
         :val [1 2 3]
         :steps [2]
-        :reason "Extra input"}]]]))
+        :reason "Extra input"}]]]
+
+    ::shape
+    {:type :circle}
+    [[[{:spec `(s/multi-spec shape-spec :type)
+        :val {:type :circle}
+        :steps []
+        :reason "no method"}]]]))


More changes on bench core ns.

More changes on bench core ns.

--- dev/bench/core.cljs
+++ dev/bench/core.cljs
@@ -1,46 +1,48 @@
 (ns bench.core
-  (:require [kdtree :as k]
+  (:require [kdtree.core :as k]
+            [cljs.pprint :refer (pprint)]
             [cljs.nodejs :as node]))

 (enable-console-print!)

+;; --- Helpers
+
 (defn generate-points
-  [n]
-  (for [x (range 0 n)
-        y (range 0 n)]
-    #js [x y]))
+  ([n] (generate-points n 1))
+  ([n step]
+   (into-array
+    (for [x (range 0 n step)
+          y (range 0 n step)]
+      #js [x y]))))

-(defn bench-init-100
-  []
-  (js/console.time "init:100")
-  (let [points (into-array (generate-points 10))]
-    (k/create2d points)
-    (js/console.timeEnd "init:100")))
+;; --- Index Initialization Bechmark

-(defn bench-init-10000
+(defn- bench-init-10000
   []
   (js/console.time "init:10000")
-  (let [points (into-array (generate-points 100))]
+  (let [points (generate-points 1000 10)]
+    (println (count points))
     (k/create2d points)
     (js/console.timeEnd "init:10000")))

-(defn bench-init-160000
+(defn- bench-init-250000
   []
-  (js/console.time "init:160000")
-  (let [points (into-array (generate-points 400))]
+  (js/console.time "init:250000")
+  (let [points (generate-points 5000 10)]
+    (println (count points))
     (k/create2d points)
-    (js/console.timeEnd "init:160000")))
+    (js/console.timeEnd "init:250000")))

 (defn bench-init
   []
-  (bench-init-100)
   (bench-init-10000)
-  (bench-init-160000))
+  (bench-init-250000))

-(defn bench-knn-160000
+;; --- Nearest Search Benchmark
+
+(defn- bench-knn-160000
   []
-  (let [tree (-> (into-array (generate-points 400))
-                 (k/create2d))]
+  (let [tree (k/create2d (generate-points 4000 10))]
     (dotimes [i 100]
       (js/console.time "knn:160000")
       (let [pt #js [(rand-int 400)
@@ -48,10 +50,9 @@
         (.nearest tree pt 2))
       (js/console.timeEnd "knn:160000"))))

-(defn bench-knn-360000
+(defn- bench-knn-360000
   []
-  (let [tree (-> (into-array (generate-points 600))
-                 (k/create2d))]
+  (let [tree (k/create2d (generate-points 6000 10))]
     (dotimes [i 100]
       (js/console.time "knn:360000")
       (let [pt #js [(rand-int 600)
@@ -64,17 +65,29 @@
   (bench-knn-160000)
   (bench-knn-360000))

+;; --- Accuracity tests
+
+(defn test-accuracity
+  []
+  (let [tree (k/create2d (generate-points 4000 20))]
+    (print "[1742 1419]")
+    (pprint (js->clj (.nearest tree #js [1742 1419] 6)))
+    (print "[1742 1420]")
+    (pprint (js->clj (.nearest tree #js [1742 1420] 6)))
+    ))
+
 (defn main
   [& [type]]
-  (bench-init)
-  (bench-knn)
-  #_(cond
+  (cond
     (= type "init")
     (bench-init)

     (= type "knn")
     (bench-knn)

+    (= type "test")
+    (test-accuracity)
+
     :else
     (println "not implemented")))


Use mapcat in problem68.clj

Use mapcat in problem68.clj

--- euler_clojure/src/euler/problem68.clj
+++ euler_clojure/src/euler/problem68.clj
@@ -33,10 +33,8 @@
 (defn problem68 []
   (let [start 6  ; Each line cannot sum to less than 6 (1+2+3)
         end 28]  ; Or greater than 27 (8+9+10)
-    (->> (map five-gon-rings (range start end))
-         (apply concat)
+    (->> (mapcat five-gon-rings (range start end))
          (filter (complement empty?))
          (map (comp str/join #(map str %) flatten))
          (filter #(= (count %) 16))
          ((comp bigint last sort)))))
-


makes the kv-store factory available to custom entitlement managers (#1232)

--- waiter/src/waiter/core.clj
+++ waiter/src/waiter/core.clj
@@ -658,8 +658,10 @@
                        router-id]
                 (discovery/register router-id curator name (str base-path "/" discovery-relative-path)
                                     {:host host :port (primary-port port)}))
-   :entitlement-manager (pc/fnk [[:settings entitlement-config]]
-                          (utils/create-component entitlement-config))
+   :entitlement-manager (pc/fnk [[:settings entitlement-config]
+                                 kv-store-factory]
+                          (let [context {:kv-store-factory kv-store-factory}]
+                            (utils/create-component entitlement-config :context context)))
    :fallback-state-atom (pc/fnk [] (atom {:available-service-ids #{}
                                           :healthy-service-ids #{}}))
    :http-client-properties (pc/fnk [[:settings [:instance-request-properties client-buffer-size client-connection-idle-timeout-ms connection-timeout-ms]]]


CMR-3048: Removed some commented code

CMR-3048: Removed some commented code

--- message-queue-lib/src/cmr/message_queue/queue/sqs.clj
+++ message-queue-lib/src/cmr/message_queue/queue/sqs.clj
@@ -325,7 +325,6 @@

 ;; Tests to make sure SNS/SQS is working
 (comment
-  ; (.subscribe (:sns-client broker) "arn:aws:sns:us-east-1:688991608580:gsfc-eosdis-cmr-ingest_exchange" "sqs" "arn:aws:sqs:us-east-1:688991608580:gsfc-eosdis-cmr-ngap-cmr-test_queue")
   ;; create a broker
   ;; (def broker (lifecycle/start (create-queue-broker {}) nil))
   (def broker (lifecycle/start (create-queue-broker {:queues ["jn_test_queueX5" "jn_test_queueY5" "jn_test_queueZ5"]
@@ -352,8 +351,6 @@
   ;; subscribe to test queue with a simple handler that prints received messages
   (queue/subscribe broker "jn_test_queueY5" (fn [msg] (do (println "MESSAGE!") (swap! msg-cnt-atom inc))))
   ;; publish a message to the queue to verify our subscribe worked
-  (queue/publish-to-queue broker "jn_test_queueY5" {:action "concept-update" :dummy "John"})
+  (queue/publish-to-queue broker "jn_test_queueY5" {:action "concept-update" :dummy "dummy"})
   ;; publish a message to the exchange to verify the message is sent to the queue
-  (queue/publish-to-exchange broker "jn_test_exchangeA5" {:action "concept-update" :dummy "Dan"})
-  (println "OK")
-  @msg-cnt-atom)
+  (queue/publish-to-exchange broker "jn_test_exchangeA5" {:action "concept-update" :dummy "dummy"}))


Remove commented-out code

Remove commented-out code

--- src/clj/weasel/repl/websocket.clj
+++ src/clj/weasel/repl/websocket.clj
@@ -127,11 +127,3 @@

 (defn- send-for-eval! [js]
   (server/send! (pr-str {:op :eval-js, :code js})))
-
-(comment
-  (let [user-env '{:ns nil :locals {}}
-        cenv (atom {})]
-    (env/with-compiler-env cenv
-      (pprint (mapcat (juxt :provides :requires)
-                 (cljsc/cljs-dependencies {} ["weasel.repl"])))))
-  )


gotten rid of zookeper methods in cluster, finally

--- storm-multicore/src/clj/backtype/storm/cluster.clj
+++ storm-multicore/src/clj/backtype/storm/cluster.clj
@@ -31,35 +31,16 @@
   (set-assignment! [this storm-id info]))

-(defn- issue-callback!
-  "Set callback to null and execute it if was not null."
-  [cb-atom]
-  (let [cb @cb-atom]
-    (reset! cb-atom nil)
-    (when cb
-      (cb))))
-
-
-(defn- issue-map-callback!
-  "Remove callback from a map and execute it if not null."
-  [cb-atom id]
-  (let [cb (@cb-atom id)]
-    (swap! cb-atom dissoc id)
-    (when cb
-      (cb id))))
-
-
 (defn mk-storm-cluster-state
   "Make state for one instance of a 'storm'."
   [conf]
   (let [
         assignment-info-callback (atom {})
-        supervisors-callback (atom nil)
         assignments-callback (atom nil)
+        storm-base-callback (atom {})
+        ;; store active storms in a map
+        active-storms (atom {})
         assignments (atom {})
-        storm-base-callback (atom {})
-        ;; store active storms in a set
-        active-storms-map (atom {})
         state-id (uuid)]
     (reify
       StormClusterState
@@ -69,25 +50,23 @@
         (when callback
           (reset! assignments-callback callback))
         (or (keys @assignments) []))
-;        (get-children cluster-state ASSIGNMENTS-SUBTREE (not-nil? callback)))

       (assignment-info
         [this storm-id callback]
         (when callback
           (swap! assignment-info-callback assoc storm-id callback))
         (@assignments storm-id))
-;        (maybe-deserialize (get-data cluster-state (assignment-path storm-id) (not-nil? callback))))

       (active-storms
         [this]
         ;; Return storm ids that are running
         ;; or empty if none are running.
-        (or (keys @active-storms-map) []))
+        (or (keys @active-storms) []))

       (activate-storm!
         [this storm-id storm-base]
         ; Map the storm id to StormBase in active map
-        (swap! active-storms-map assoc storm-id storm-base))
+        (swap! active-storms assoc storm-id storm-base))

       (update-storm!
         [this storm-id new-elems]
@@ -96,7 +75,7 @@
               executors (:component->executors base)
               ;; new-elems' :component->executors map is updated with merge
               new-elems (update new-elems :component->executors (partial merge executors))]
-          (swap! active-storms-map assoc storm-id (-> base (merge new-elems)))))
+          (swap! active-storms assoc storm-id (-> base (merge new-elems)))))

       (storm-base
         [this storm-id callback]
@@ -104,88 +83,19 @@
         ; which will be ran when change is made
         (when callback
           (swap! storm-base-callback assoc storm-id callback))
-        (@active-storms-map storm-id))
+        (@active-storms storm-id))

       (remove-storm-base!
         [this storm-id]
-        (swap! active-storms-map dissoc storm-id))
-;        (delete-node cluster-state (storm-path storm-id)))
+        (swap! active-storms dissoc storm-id))

       (set-assignment!
         [this storm-id info]
+        ;; associates an assignment with a topology
         (swap! assignments assoc storm-id info))
-;        (set-data cluster-state (assignment-path storm-id) (Utils/serialize info)))

       (remove-storm!
         [this storm-id]
+        ;; removes assignments and stormbase of a topology
         (swap! assignments dissoc storm-id)
-;        (delete-node cluster-state (assignment-path storm-id))
-        (remove-storm-base! this storm-id)))))
-
-
-;; daemons have a single thread that will respond to events
-;; start with initialize event
-;; callbacks add events to the thread's queue
-
-;; keeps in memory cache of the state, only for what client subscribes to. Any subscription is automatically kept in sync, and when there are changes, client is notified.
-;; master gives orders through state, and client records status in state (ephemerally)
-
-;; master tells nodes what workers to launch
-
-;; master writes this. supervisors and workers subscribe to this to understand complete topology. each storm is a map from nodes to workers to tasks to ports whenever topology changes everyone will be notified
-;; master includes timestamp of each assignment so that appropriate time can be given to each worker to start up
-;; /assignments/{storm id}
-
-;; which tasks they talk to, etc. (immutable until shutdown)
-;; everyone reads this in full to understand structure
-;; /tasks/{storm id}/{task id} ; just contains bolt id
-
-;; supervisors send heartbeats here, master doesn't subscribe but checks asynchronously
-;; /supervisors/status/{ephemeral node ids}  ;; node metadata such as port ranges are kept here
-
-;; tasks send heartbeats here, master doesn't subscribe, just checks asynchronously
-;; /taskbeats/{storm id}/{ephemeral task id}
-
-;; contains data about whether it's started or not, tasks and workers subscribe to specific storm here to know when to shutdown
-;; master manipulates
-;; /storms/{storm id}
-
-;; Zookeeper flows:
-
-;; Master:
-;; job submit:
-;; 1. read which nodes are available
-;; 2. set up the worker/{storm}/{task} stuff (static)
-;; 3. set assignments
-;; 4. start storm - necessary in case master goes down, when goes back up can remember to take down the storm (2 states: on or off)
-
-;; Monitoring (or by checking when nodes go down or heartbeats aren't received):
-;; 1. read assignment
-;; 2. see which tasks/nodes are up
-;; 3. make new assignment to fix any problems
-;; 4. if a storm exists but is not taken down fully, ensure that storm takedown is launched (step by step remove tasks and finally remove assignments)
-
-;; masters only possible watches is on ephemeral nodes and tasks, and maybe not even
-
-;; Supervisor:
-;; 1. monitor /storms/* and assignments
-;; 2. local state about which workers are local
-;; 3. when storm is on, check that workers are running locally & start/kill if different than assignments
-;; 4. when storm is off, monitor tasks for workers - when they all die or don't hearbeat, kill the process and cleanup
-
-;; Worker:
-;; 1. On startup, start the tasks if the storm is on
-
-;; Task:
-;; 1. monitor assignments, reroute when assignments change
-;; 2. monitor storm (when storm turns off, error if assignments change) - take down tasks as master turns them off
-
-;; locally on supervisor: workers write pids locally on startup, supervisor deletes it on shutdown (associates pid with worker name)
-;; supervisor periodically checks to make sure processes are alive
-;; {rootdir}/workers/{storm id}/{worker id}   ;; contains pid inside
-
-;; all tasks in a worker share the same cluster state
-;; workers, supervisors, and tasks subscribes to storm to know when it's started or stopped
-;; on stopped, master removes records in order (tasks need to subscribe to themselves to see if they disappear)
-;; when a master removes a worker, the supervisor should kill it (and escalate to kill -9)
-;; on shutdown, tasks subscribe to tasks that send data to them to wait for them to die. when node disappears, they can die
+        (remove-storm-base! this storm-id)))))