use clojure java interop to extract resources rather than external unzip command

--- storm-core/src/clj/backtype/storm/util.clj
+++ storm-core/src/clj/backtype/storm/util.clj
@@ -24,7 +24,7 @@
   (:import [java.util.zip ZipFile])
   (:import [java.util.concurrent.locks ReentrantReadWriteLock])
   (:import [java.util.concurrent Semaphore])
-  (:import [java.io File RandomAccessFile StringWriter PrintWriter])
+  (:import [java.io File FileOutputStream StringWriter PrintWriter IOException])
   (:import [java.lang.management ManagementFactory])
   (:import [org.apache.commons.exec DefaultExecutor CommandLine])
   (:import [org.apache.commons.io FileUtils])
@@ -33,6 +33,7 @@
   (:require [clojure [string :as str]])
   (:import [clojure.lang RT])
   (:require [clojure [set :as set]])
+  (:require [clojure.java.io :as io])
   (:use [clojure walk])
   (:use [backtype.storm log])
   )
@@ -366,8 +367,13 @@

 (defn extract-dir-from-jar [jarpath dir destdir]
   (try-cause
-    (exec-command! (str "unzip -qq " jarpath " " dir "/** -d " destdir))
-  (catch ExecuteException e
+    (with-open [jarpath (ZipFile. jarpath)]
+      (let [entries (enumeration-seq (.entries jarpath))]
+        (doseq [file (filter (fn [entry](and (not (.isDirectory entry)) (.startsWith (.getName entry) dir))) entries)]
+          (.mkdirs (.getParentFile (File. destdir (.getName file))))
+          (with-open [out (FileOutputStream. (File. destdir (.getName file)))]
+            (io/copy (.getInputStream jarpath file) out)))))
+  (catch IOException e
     (log-message "Could not extract " dir " from " jarpath))
   ))


CMR-2414 Fix some issues with GorillaREPL for Sprint 46 and fix some namespacing.

--- umm-lib/src/cmr/umm/dif10/collection.clj
+++ umm-lib/src/cmr/umm/dif10/collection.clj
@@ -117,8 +117,8 @@
   (for [citation (:collection-citations collection)
         :let [other-reference-details (:other-reference-details citation)]
         :when (not (nil? other-reference-details))]
-    (clojure.data.xml/element :Dataset_Citation {}
-                              (clojure.data.xml/element :Other_Citation_Details {} other-reference-details))))
+    (x/element :Dataset_Citation {}
+               (x/element :Other_Citation_Details {} other-reference-details))))

 (extend-protocol dif10-core/UmmToDif10Xml
   UmmCollection


improve callback

improve callback

--- src/node/server/nexmo.cljs
+++ src/node/server/nexmo.cljs
@@ -7,7 +7,7 @@
    [cljs.nodejs :as nodejs]
    [app.debug :refer [echo]]))

-(def nexmo (nodejs/require "easynexmo"))
+(def nexmo (nodejs/require "easynexmo")) ; https://github.com/pvela/nexmo

 (def nexmo-key (env :nexmo-key))
 (def nexmo-secret (env :nexmo-secret))
@@ -17,22 +17,24 @@
   (assert nexmo-secret)
   (.initialize nexmo nexmo-key nexmo-secret "https"))

-(defn put1! [ch val]
+(defn collect [ch err val]
+  (when err
+    (println "[NEXMO]" err))
   (when val
-    (put! ch val))
+    (put! ch (js->clj val)))
   (close! ch))

 (defn get-numbers [account]
   (let [result (chan)]
     (go
-      (.getNumbers account #(put1! result %)))
+      (.getNumbers account (partial collect result)))
     result))

 ;; (echo (get-numbers nexmo))

 (defn send-notification []
   (let [response (chan)]
-    (.sendTTSMessage nexmo "16502913436" "Shuttle Offline" #js {} #(put1! response %))
+    (.sendTTSMessage nexmo "16502913436" "Shuttle Offline" #js {} (partial collect response))
     response))

 ;; (echo (send-notification))


Unit tests for add-vary-header

Unit tests for add-vary-header

--- test/vignette/util/image_response_test.clj
+++ test/vignette/util/image_response_test.clj
@@ -41,3 +41,11 @@
        (add-surrogate-header {} {:wikia "wikia" :original "orig" :image-type "images"}) =>
        {:headers {"Surrogate-Key"   "c8dfba77e9beb5c26ca20d4411674065d4a0ded5"
                   "X-Surrogate-Key" "c8dfba77e9beb5c26ca20d4411674065d4a0ded5"}})
+
+(facts :add-vary-header
+       (add-vary-header {} {:requested-format nil :request-type :thumbnail :original "some-file.png" :image-type "images"}) => {:headers {"Vary" "Accept"}}
+       (add-vary-header {} {:request-type :thumbnail :original "some-file.png" :image-type "images"}) => {:headers {"Vary" "Accept"}}
+       (add-vary-header {} {:requested-format nil :request-type :original :original "some-file.png" :image-type "images"}) => {}
+       (add-vary-header {} {:requested-format nil :request-type :thumbnail :original "some-file.bmp" :image-type "images"}) => {}
+       (add-vary-header {} {:requested-format "png" :request-type :thumbnail :original "some-file.png" :image-type "images"}) => {})
+


support mapping java.util.UUID to one of our base field types.

--- src/metabase/driver.clj
+++ src/metabase/driver.clj
@@ -40,6 +40,7 @@
    java.sql.Date                :DateField
    java.sql.Timestamp           :DateTimeField
    java.util.Date               :DateField
+   java.util.UUID               :TextField
    org.postgresql.util.PGobject :UnknownField}) ; this mapping included here since Native QP uses class->base-type directly. TODO - perhaps make *class-base->type* driver specific?

 ;; ## Driver Lookup


Make clear that the user is responsible for providing the corrent number of validations.

--- src/active/clojure/validation.cljc
+++ src/active/clojure/validation.cljc
@@ -185,7 +185,11 @@
   they were validated.

   If any one validation fails, returns a `ValidationFailure`,
-  containing _all_ failures."
+  containing _all_ failures.
+
+  The number of arguments `make-result` expects must match the `(count
+  validations`).  Supplying a wrong number of arguments considered
+  undefined behaviour."
   [make-result & validations]
   (reduce (fn [res v] (seq-validation res v))
           (pure-validation (curry-n make-result (count validations)))


deref typo in tags

deref typo in tags

--- src/braid/ui/views/pages/tags.cljs
+++ src/braid/ui/views/pages/tags.cljs
@@ -56,7 +56,7 @@
           [new-tag-view {:group-id @group-id}]

           (let [subscribed-tags
-                (->> @storted-tags
+                (->> @sorted-tags
                      (filter (fn [t] (@subscribed-to-tag? (t :id)))))]
             (when (seq subscribed-tags)
               [:div


[ref] Simplify where clause

[ref] Simplify where clause

--- src/ombs/db/payment.clj
+++ src/ombs/db/payment.clj
@@ -16,12 +16,12 @@
   "sum of all credits and debits in pays for current user and event. "
   ; full debt
   ([username]
-   (if-let [summary (:debt (first (sql/select debts (sql/where (= username :user)) (sql/aggregate (sum :debt) :debt))))]
+   (if-let [summary (:debt (first (sql/select debts (sql/where {:user username}) (sql/aggregate (sum :debt) :debt))))]
      summary
      0.0))
   ; debt on some event
   ([username event date]
-   (if-let [summary (:debt (first (sql/select debts (sql/where (and (= username :user) (= event :event) (= date :date))))))]
+   (if-let [summary (:debt (first (sql/select debts (sql/where {:user username :event event :date date}))))]
      summary
      0.0))
   ([uid eid]


Spend event id on page.

Spend event id on page.

--- src/ombs/view/event.clj
+++ src/ombs/view/event.clj
@@ -16,14 +16,14 @@
   [:button] button)

 (declare get-action)
-(h/defsnippet event-elem "../resources/public/event.html" [:.event] [{:keys [name price date author status parts] :as event}]
-  [:.event2] (h/set-attr :id 1)
+(h/defsnippet event-elem "../resources/public/event.html" [:.event] [{:keys [id name price date author status parts] :as event}]
+  [:.event] (h/set-attr :id id)
   [:.name]   (h/content (str author "'s " name))
   [:.date]   (h/content (str date))
   [:.action] (partial action (get-action event)))

 (defn get-button [button-type {:keys [name price date author status parts]}]
-  (println button-type)
+  ;(println button-type)
   (case button-type
     :pay          (partial actions/pay)
     :start        (partial actions/start)
@@ -34,9 +34,9 @@

 (defn get-action [{:keys [name price date author status parts] :as event}]
   "return hashmap with function for money and button"
-  (println event)
-  (println (core/participated? "Intey" name date))
-  (println (core/participants-count name date))
+  ;(println event)
+  ;(println (core/participated? "Intey" name date))
+  ;(println (core/participants-count name date))
   (if-let [uname (sess/get :username)]
     (case status
       "initial" (cond


Modify AudioBus and ControlBus print formatter to print a default name and to label the attributes more clearly.

--- src/overtone/sc/bus.clj
+++ src/overtone/sc/bus.clj
@@ -34,8 +34,10 @@
   (free-bus [this] (free-id :control-bus (:id this) (:n-channels this))))

 (defmethod print-method AudioBus [b w]
-  (.write w (format "#<audio-bus: %s %s %d>"
-                    (:name b)
+  (.write w (format "#<audio-bus: %s, %s, id %d>"
+                    (if (empty? (:name b))
+                      "No Name"
+                      (:name b))
                     (cond
                      (= 1 (:n-channels b)) "mono"
                      (= 2 (:n-channels b)) "stereo"
@@ -43,8 +45,10 @@
                     (:id b))))

 (defmethod print-method ControlBus [b w]
-  (.write w (format "#<control-bus: %s %s %d>"
-                    (:name b)
+  (.write w (format "#<control-bus: %s, %s, id %d>"
+                    (if (empty? (:name b))
+                      "No Name"
+                      (:name b))
                     (cond
                      (= 1 (:n-channels b)) "1 channel"
                      :else (str (:n-channels b) " channels"))


Add test for synthetic nonce

Add test for synthetic nonce

--- test/caesium/magicnonce/secretbox_test.clj
+++ test/caesium/magicnonce/secretbox_test.clj
@@ -87,6 +87,14 @@
       (is (u/array-eq c1 c2))
       (is (= (range s/noncebytes) (take s/noncebytes c1) (take s/noncebytes c2))))))

+(deftest synthetic-nonce-test
+  (is (= (alength ^bytes (#'ms/synthetic-nonce (byte-array (range 10))))
+         (s/noncebytes)))
+  (is (= (#'ms/synthetic-nonce (byte-array (range 10)))
+         (#'ms/synthetic-nonce (byte-array (range 10)))))
+  (is (not= (#'ms/synthetic-nonce (byte-array (range 10)))
+            (#'ms/synthetic-nonce (byte-array (range 11))))))
+
 (defn repeated-keystream?
   "Does given scheme repeat the keystream when applied to given
   plaintexts?


Update ClojureScript dep to 2138

Update ClojureScript dep to 2138

--- src/leiningen/new/mies/project.clj
+++ src/leiningen/new/mies/project.clj
@@ -3,7 +3,7 @@
   :url "http://example.com/FIXME"

   :dependencies [[org.clojure/clojure "1.5.1"]
-                 [org.clojure/clojurescript "0.0-2127"]]
+                 [org.clojure/clojurescript "0.0-2138"]]

   :plugins [[lein-cljsbuild "1.0.1"]]


replace standard lein JVM opts

replace standard lein JVM opts

--- src/leiningen/new/mies/project.clj
+++ src/leiningen/new/mies/project.clj
@@ -3,6 +3,7 @@
   :url "http://example.com/FIXME"
   :dependencies [[org.clojure/clojure "1.6.0"]
                  [org.clojure/clojurescript "0.0-2850"]]
+  :jvm-opts ^:replace ["-Xmx1g" "-server"]
   :node-dependencies [[source-map-support "0.2.8"]]
   :plugins [[lein-npm "0.4.0"]]
   :source-paths ["src" "target/classes"]


Versionumeron korotus

Versionumeron korotus

--- mtg-pairings-wer/src/mtg_pairings/core.clj
+++ mtg-pairings-wer/src/mtg_pairings/core.clj
@@ -18,7 +18,7 @@
            javax.swing.table.TableRowSorter
            org.joda.time.LocalDate))

-(def version "0.2.0")
+(def version "0.2.1")

 (defonce state (atom {:properties {}}))


defr macro now converts dashed-names in props to camelCase

--- src/pump/def_macros.clj
+++ src/pump/def_macros.clj
@@ -1,5 +1,20 @@
-(ns pump.def-macros)
+(ns pump.def-macros
+  (:require [clojure.string :as string]))
+
+(defn dash-to-camel-name
+  [name]
+  (let [words (string/split name #"-")
+        camels (map string/capitalize (rest words))
+        complete (apply str (first words) camels)]
+    complete))
+
+(defn dash-to-camel-keys
+  [hashmap]
+  (into {} (for [[k v] hashmap]
+             [(keyword (dash-to-camel-name (name k))) v])))

 (defmacro defr
   [name body]
-  `(def ~name (pump.core/react (assoc ~body :displayName name))))
+  (let [cameled# (dash-to-camel-keys body)
+        body# (assoc cameled# :displayName name)]
+    `(def ~name (pump.core/react ~body#))))


ns should match filename

ns should match filename

--- test/onyx/log/grouping_generative_test.clj
+++ test/onyx/log/grouping_generative_test.clj
@@ -1,4 +1,4 @@
-(ns onyx.log.grouping-generative-task
+(ns onyx.log.grouping-generative-test
   (:require [onyx.messaging.dummy-messenger :refer [->DummyMessenger]]
             [onyx.log.generators :as log-gen]
             [onyx.extensions :as extensions]


use clojure.set directly

use clojure.set directly

--- src/riemann/streams.clj
+++ src/riemann/streams.clj
@@ -16,7 +16,7 @@
   (:require [riemann.periodic :as periodic])
   (:require riemann.client)
   (:require riemann.logging)
-  (:require [clojure.set])
+  (:require [clojure.set :as set])
   (:use [clojure.math.numeric-tower])
   (:use [clojure.tools.logging]))

@@ -604,7 +604,7 @@
   (tagged-all [\"foo\" \"bar\"] prn)"
   [tags & children]
   (fn [event]
-    (when (subset? tags (:tags event))
+    (when (set/subset? (set tags) (set (:tags event)))
       (call-rescue event children))))
 (def tagged tagged-all)


matches all fields from scheduler-config and the KubernetesScheduler instance (#1217)

--- waiter/src/waiter/scheduler/kubernetes.clj
+++ waiter/src/waiter/scheduler/kubernetes.clj
@@ -627,6 +627,7 @@
                                 pod-suffix-length
                                 replicaset-api-version
                                 replicaset-spec-builder-fn
+                                restart-expiry-threshold
                                 retrieve-auth-token-state-fn
                                 retrieve-syncer-state-fn
                                 reverse-proxy
@@ -1384,6 +1385,7 @@
                                            pod-suffix-length
                                            replicaset-api-version
                                            replicaset-spec-builder-fn
+                                           restart-expiry-threshold
                                            retrieve-auth-token-state-fn
                                            retrieve-syncer-state-fn
                                            reverse-proxy
@@ -1396,4 +1398,6 @@
           rs-watch-thread (start-replicasets-watch! scheduler watch-options)]
       (reset! daemon-state {:pod-watch-daemon pod-watch-thread
                             :rs-watch-daemon rs-watch-thread})
+      (assert (every? #(contains? scheduler %) (keys scheduler-config))
+              "ensure all fields in scheduler-config are present in KubernetesScheduler")
       scheduler)))


rename locals

rename locals

--- src/sicp/chapter3_3.clj
+++ src/sicp/chapter3_3.clj
@@ -479,12 +479,12 @@

 ;; Exercise 3.17
 (defn count-pairs' [x]
-  (let [s (atom [])
+  (let [memo (atom [])
         f (fn f [x]
             (if (or (not (pair? x))
-                    (some #(identical? x %) @s))
+                    (some #(identical? x %) @memo))
               0
-              (do (swap! s conj x)
+              (do (swap! memo conj x)
                   (+ (f (car x))
                      (f (cdr x))
                      1))))]
@@ -492,12 +492,12 @@

 ;; Exercise 3.18
 (defn contains-cycle? [x]
-  (let [s (atom [])
+  (let [memo (atom [])
         f (fn f [x]
             (cond
               (not (pair? (cdr x))) false
-              (some #(identical? x %) @s) true
-              :else (do (swap! s conj x)
+              (some #(identical? x %) @memo) true
+              :else (do (swap! memo conj x)
                         (recur (cdr x)))))]
     (f x)))


put full state to cache

put full state to cache

--- src/syncserver/core.clj
+++ src/syncserver/core.clj
@@ -18,7 +18,6 @@
 (defmacro debug [& text]
   `(when @debug-mode (println ~@text)))

-
 (def cache! (.. (CacheBuilder/newBuilder) (expireAfterAccess 500 TimeUnit/SECONDS) (build)))

 (def current-state
@@ -29,7 +28,7 @@
   (swap! current-state
          (fn [cs]
            (let [nk (inc (:current cs))]
-             (.put cache! (str nk) state)
+             (.put cache! (str nk) {:state state :current nk})
              (debug "Cached" (.getIfPresent cache! (str nk)) "at" (str nk))
              (assoc cs :state state :current nk)))))

@@ -105,6 +104,7 @@

 (defn- compute-delta [os cs]
+  (debug os cs)
   (object-array [(:current cs) (map-diff [] (:state os) (:state cs) #{})]))

 (defn- get-from-cache [old-state]


Updating dev system user to allow easy use of CMR SOAP Services.

--- dev-system/dev/user.clj
+++ dev-system/dev/user.clj
@@ -45,10 +45,22 @@
   (println "Logging level set to" level)
   nil)

+(defn configure-for-soap-services
+  []
+  (config/set-config-value! :echo-rest-host "localhost")
+  (config/set-config-value! :echo-rest-port 3012)
+  (config/set-config-value! :echo-rest-context "/soap-services/rest")
+  (config/set-config-value! :echo-system-token "mock-echo-system-token")
+  (config/set-config-value! :dev-system-echo-type "external")
+  (config/set-config-value! :dev-system-db-type "in-memory"))
+
 (defn start
   "Starts the current development system."
   []
   (config/reset-config-values)
+
+  ;; Uncomment this to force CMR to use SOAP Services
+  ; (configure-for-soap-services)

   (jobs/set-default-job-start-delay! (* 3 3600))


CMR-1288: more refactoring.

CMR-1288: more refactoring.

--- umm-lib/src/cmr/umm/validation/granule.clj
+++ umm-lib/src/cmr/umm/validation/granule.clj
@@ -42,27 +42,35 @@
        [(format "%%s have [%s] which do not reference any projects in parent collection."
                 (str/join ", " missing-project-refs))]})))

-(defn matches-collection-identifier-validation
+(defn- matches-collection-identifier-validation
   "Validates the granule collection-ref field matches the corresponding field in the parent collection."
-  [field parent-field-path]
-  (fn [_ granule]
-    (let [value (get-in granule [:collection-ref field])
-          parent-value (get-in granule (concat [:parent] parent-field-path))
-          field-name (name field)]
-      (when (and value (not= value parent-value))
-        {[:collection-ref]
-         [(format "%%s %s [%s] does not match the %s of the parent collection [%s]"
-                  field-name value field-name parent-value)]}))))
+  [granule field parent-field-path]
+  (let [value (get-in granule [:collection-ref field])
+        parent-value (get-in granule (concat [:parent] parent-field-path))
+        field-name (name field)]
+    (when (and value (not= value parent-value))
+      {[:collection-ref]
+       [(format "%%s %s [%s] does not match the %s of the parent collection [%s]"
+                field-name value field-name parent-value)]})))

-(defn collection-ref-validation
-  "Validates the granules collection ref matches the parent collection."
-  [_ granule]
+(defn- collection-ref-required-fields-validation
+  "Validates the granules collection ref has the required fields."
+  [granule]
   (let [{:keys [collection-ref]} granule
         {:keys [short-name version-id entry-title]} collection-ref]
     (when-not (or entry-title (and short-name version-id))
       {[:collection-ref]
        [(format "%%s should have at least entry-title or short-name and version-id, but was %s"
                 (util/remove-nil-keys (into {} collection-ref)))]})))
+
+(defn- collection-ref-validation
+  "Validates the granules collection ref"
+  [_ granule]
+  (merge-with concat
+              (collection-ref-required-fields-validation granule)
+              (matches-collection-identifier-validation granule :entry-title [:entry-title])
+              (matches-collection-identifier-validation granule :short-name [:product :short-name])
+              (matches-collection-identifier-validation granule :version-id [:product :version-id])))

 (defn- temporal-error-message
   "Returns an error message for given pairs of granule and collection start and end dates."
@@ -131,9 +139,6 @@
 (def granule-validations
   "Defines validations for granules"
   [collection-ref-validation
-   (matches-collection-identifier-validation :entry-title [:entry-title])
-   (matches-collection-identifier-validation :short-name [:product :short-name])
-   (matches-collection-identifier-validation :version-id [:product :version-id])
    {:spatial-coverage spatial-coverage-validations
     :temporal temporal-validation
     :platform-refs [(vu/unique-by-name-validator :short-name)


CIRCLE-2526 Logged out user can see oss builds (#1612)

* CIRCLE-2526 Logged out user can see oss builds

* add comments

* a better comment

* add more comments

--- src-cljs/frontend/components/dashboard.cljs
+++ src-cljs/frontend/components/dashboard.cljs
@@ -136,9 +136,13 @@
             builds-per-page (:builds-per-page data)
             current-user (:current-user data)]
         (html
-         (cond (or (nil? builds)
-                   (nil? projects))
-               [:div.empty-placeholder (spinner)]
+          ;; ensure the both projects and builds are loaded before rendering to prevent
+          ;; the build list and branch picker from resizing.
+          (cond (or (nil? builds)
+                    ;; if the user isn't logged in, but is viewing an oss build,
+                    ;; then we will not load any projects.
+                    (and current-user (nil? projects)))
+                [:div.empty-placeholder (spinner)]

                (and (empty? builds)
                     projects


Firefox treats 224 as Cmd

Firefox treats 224 as Cmd

--- src-cljs/frontend/components/key_queue.cljs
+++ src-cljs/frontend/components/key_queue.cljs
@@ -45,7 +45,8 @@
    187 "="
    189 "-"
    190 "."
-   220 "\\"})
+   220 "\\"
+   224 "meta"})

 (defn event-modifiers
   "Given a keydown event, return the modifier keys that were being held."


notify after chat is submitted

notify after chat is submitted

--- src-cljs/frontend/controllers/controls.cljs
+++ src-cljs/frontend/controllers/controls.cljs
@@ -1299,7 +1299,8 @@
                                                :chat/document (:document/id previous-state)
                                                :client/timestamp (datetime/server-date)
                                                ;; server will overwrite this
-                                               :server/timestamp (datetime/server-date)})]))))
+                                               :server/timestamp (datetime/server-date)})]))
+    (maybe-notify-subscribers! previous-state current-state nil nil)))

 (defmethod control-event :chat-toggled
   [browser-state message _ state]


remove comms from state

remove comms from state

- references pull #1392

--- src-cljs/frontend/controllers/controls.cljs
+++ src-cljs/frontend/controllers/controls.cljs
@@ -501,8 +501,8 @@

 (defmethod post-control-event! :merge-pull-request-clicked
-  [target message {:keys [vcs-url owner repo number sha] :as args} previous-state current-state]
-  (let [api-ch (-> current-state :comms :api)
+  [target message {:keys [vcs-url owner repo number sha] :as args} previous-state current-state comms]
+  (let [api-ch (:api comms)
         vcs-type (vcs-url/vcs-type vcs-url)
         uuid frontend.async/*uuid*]
     (go


removes println (#1018)

removes println (#1018)

--- waiter/test/waiter/scheduler/kubernetes_test.clj
+++ waiter/test/waiter/scheduler/kubernetes_test.clj
@@ -1651,7 +1651,6 @@
     (testing "pod to live instance"
       (let [dummy-scheduler (assoc base-scheduler :restart-expiry-threshold 10)
             instance (pod->ServiceInstance dummy-scheduler pod)]
-        (println 'INSTANCE instance)
         (is (= (scheduler/make-ServiceInstance instance-map) instance))))

     (testing "pod with expired annotation"


Clarify this test somewhat

Clarify this test somewhat

--- web/test/immutant/web/websocket_test.clj
+++ web/test/immutant/web/websocket_test.clj
@@ -95,27 +95,20 @@

 (deftest share-session-with-websocket
   (let [result (promise)
-        shared (atom nil)
-        handler (fn [{s :session}]
-                  (if-let [id (:id s)]
-                    (response (get @shared id))
-                    (let [id (rand)]
-                      (reset! shared {id "identified"})
-                      (-> (response "authorized")
-                        (assoc :session {:id id})))))]
+        handler (fn [{{:keys [id] :or {id (str (rand))}} :session}]
+                  (-> id response (assoc :session {:id id})))]
     (run (wrap-websocket (wrap-session handler)
-           :on-open (fn [ch hs]
-                      (let [s (session hs)]
-                        (reset! shared {(:id s) "hacked"})
-                        (deliver result s)))))
-    (is (= "authorized" (get-body url :cookies nil)))
-    (is (= "identified" (get-body url)))
-    (with-open [client (http/create-client)
-                socket (http/websocket client "ws://localhost:8080"
-                         :cookies @testing.web/cookies)]
-      (let [ring-session (deref result 1000 nil)]
-        (is (:id ring-session))))
-    (is (= "hacked" (get-body url)))
+           :on-open (fn [ch hs] (deliver result (:id (session hs))))))
+    ;; establish the id in the session with the first request
+    (let [id (get-body url :cookies nil)]
+      ;; make sure we get it again if we pass the returned cookie
+      (is (= id (get-body url)))
+      ;; now open a websocket connection with the same cookie
+      (with-open [client (http/create-client)
+                  socket (http/websocket client "ws://localhost:8080"
+                           :cookies @testing.web/cookies)]
+        ;; and verify the websocket sees the same id
+        (is (= id (deref result 1000 :failure)))))
     (stop)))

 (deftest session-invalidation


CMR-5495 Fix issue with translation endpoint causing 500 error.

--- ingest-app/src/cmr/ingest/api/translation.clj
+++ ingest-app/src/cmr/ingest/api/translation.clj
@@ -123,7 +123,9 @@
   (let [supported-input-formats (concept-type->supported-input-formats concept-type)
         supported-output-formats (concept-type->supported-output-formats concept-type)
         content-type (get headers "content-type")
-        accept-header (get headers "accept")
+        ;; CMR-5495 If we do not get an accept header, we would like to show the user an error
+        ;; describing the problem.
+        accept-header (or (get headers "accept") "")
         ;; always skip sanitize of granules for now as we have not considered sanitizing for granules yet
         skip-sanitize-umm? (= "true" (get headers "cmr-skip-sanitize-umm-c"))
         options (if (and skip-sanitize-umm? (mt/umm-json? accept-header))
@@ -132,7 +134,13 @@

     ;; just for validation (throws service error if invalid media type is given)
     (mt/extract-header-mime-type supported-input-formats headers "content-type" true)
-    (mt/extract-header-mime-type supported-output-formats headers "accept" true)
+    (mt/extract-header-mime-type
+     supported-output-formats
+     ;; CMR-5495 Put "" string back into the accept header if none is supplied to give proper
+     ;; error message.
+     (assoc headers "accept" accept-header)
+     "accept"
+     true)

     ;; Can not skip-sanitize-umm when the target format is not UMM-C
     (when (and skip-sanitize-umm? (not (mt/umm-json? accept-header)))


add alias for index

add alias for index

--- src/wb_es/bulk/core.clj
+++ src/wb_es/bulk/core.clj
@@ -105,7 +105,7 @@
               :or {db (d/db datomic-conn)}}]
   (let [index-id (format "%s_tmp" release-id)]
     (do
-      (create-index index-id)
+      (create-index index-id :default-index true)
       (let [n-threads 4
             scheduler (chan n-threads)
             logger (chan n-threads)]


Remove unnecessary (do) statements

Remove unnecessary (do) statements

--- src/sinusoides/routes.cljs
+++ src/sinusoides/routes.cljs
@@ -8,15 +8,13 @@
   (:import goog.History))

 (defn make-routes! [app]
-  (do
-    (defroute root "/" [] (om/update! app :view [:main]))))
+  (defroute root "/" [] (om/update! app :view [:main])))

 (defn init-history! []
-  (do
-    (secretary/set-config! :prefix "#")
-    (doto (History.)
-      (goog.events/listen "navigate" #(secretary/dispatch! (.-token %)))
-      (.setEnabled true))))
+  (secretary/set-config! :prefix "#")
+  (doto (History.)
+    (goog.events/listen "navigate" #(secretary/dispatch! (.-token %)))
+    (.setEnabled true)))

 (defn router [app owner]
   (reify
@@ -26,7 +24,6 @@
     (render [_] (dom/div nil))))

 (defn init-router! [state]
-  (do
-    (om/root router state
-      {:target (.getElementById js/document "router")})
-    (init-history!)))
+  (om/root router state
+    {:target (.getElementById js/document "router")})
+  (init-history!))


Refactoring :requires

Refactoring :requires

--- lambdalf-lib/src/main/clojure/alfresco/auth.clj
+++ lambdalf-lib/src/main/clojure/alfresco/auth.clj
@@ -3,7 +3,9 @@
             AuthenticationUtil
             AuthenticationUtil$RunAsWork]))

+; TODO memoize the result
 (defn admin
+  "Returns the current Administrator user name."
   []
   (AuthenticationUtil/getAdminUserName))

@@ -15,6 +17,13 @@
                                ~f))]
      (AuthenticationUtil/runAs work# ~user)))

+(defmacro run-as-fn
+  "Returns a closure which will run with the provided user privileges"
+  [user f]
+  `(fn []
+     (run-as ~user
+             ~f)))
+
 (defmacro as-admin
   "Runs the provided form as admin"
   [f]


track-cycle for better error reporting

track-cycle for better error reporting

--- src/smallex/reductions.clj
+++ src/smallex/reductions.clj
@@ -29,10 +29,22 @@

 (defn- track-cycle
   "Returns a vector with the elements of _some_ cycle which contains root. Root
-  will be both first and last element."
+  will be both first and last element of the cycle."
   [graph root]
-  ;; TODO
-  )
+  ;; Since we actually might end up in another cycle, we must backtrack if we've
+  ;; found a vertex we've already visited.
+  (letfn [(dfs [cycle-list visited to-visit]
+            (cond (= to-visit root) (conj cycle-list to-visit)
+                  (visited to-visit) nil
+                  :else
+                  (let [cycle-list (conj cycle-list to-visit)
+                        visited (conj visited to-visit)]
+                    (first
+                     (->> (map #(dfs cycle-list visited %) (get graph to-visit))
+                          (remove nil?))))))]
+    (first
+     (->> (map #(dfs [root] #{} %) (get graph root))
+          (remove nil?)))))

 (defn- toposort-transposed
   "Returns a topological sort of transposed graph, and throws an exception if


truncate by default

truncate by default

--- src/babel/parse.cljc
+++ src/babel/parse.cljc
@@ -13,7 +13,7 @@

 ;; for now, using a language-independent tokenizer.
 (def tokenizer #"[ ']")
-(def map-fn (if true map #?(:clj pmap) #?(:cljs map)))
+(def map-fn #?(:clj pmap) #?(:cljs map))

 ;; thanks to http://tobiasbayer.com/post/2014-11-12-using-clojures-core-dot-cache/
 (defn deal-with-cache [k if-miss-do lexical-cache]
@@ -242,7 +242,7 @@
   the entire tree costs about 20% time than truncating it."
   ([input model & {:keys [parse-with-truncate original-input]}]
    (let [parse-with-truncate
-         (cond true false (= parse-with-truncate false)
+         (cond (= parse-with-truncate false)
                false
                true true)
          original-input (if original-input original-input input)


Add new converter: `from-publisher`.

Add new converter: `from-publisher`.

--- src/beicon/core.cljc
+++ src/beicon/core.cljc
@@ -602,6 +602,13 @@
   #?(:cljs (.fromPromise Observable p)
      :clj (Observable/fromFuture ^Future p)))

+#?(:clj
+   (defn from-publisher
+     "Converts an arbitrary Reactive-Streams Publisher into a
+     Flowable if not already a Flowable."
+     [v]
+     (Flowable/fromPublisher ^Publisher v)))
+
 (defn just
   "Returns an observable sequence that contains
   a single element."


reader: fix read-unicode-char

reader: fix read-unicode-char

--- src/blind/reader.clj
+++ src/blind/reader.clj
@@ -329,19 +329,18 @@
      (let [l (+ offset length)]
        (when-not (== (count token) l)
          (throw (IllegalArgumentException. (str "Invalid unicode character: \\" token))))
-       (loop [uc 0 i offset]
+       (loop [i offset uc 0]
          (if (== i l)
            (char uc)
            (let [d (Character/digit ^char (nth token i) ^int base)]
              (if (== d -1)
                (throw (IllegalArgumentException. (str "Invalid digit: " (nth token i))))
-               (recur (long (* uc (+ base d))) (inc i))))))))
+               (recur (inc i) (long (+ d (* uc base))))))))))

   ([rdr initch base length exact?]
-     (let [uc (Character/digit ^char initch ^int base)]
+     (loop [i 1 uc (Character/digit ^char initch ^int base)]
        (if (== uc -1)
-         (throw (IllegalArgumentException. (str "Invalid digit: " initch))))
-       (loop [i 1 uc uc]
+         (throw (IllegalArgumentException. (str "Invalid digit: " initch)))
          (if-not (== i length)
            (let [ch (peek-char rdr)]
              (if (or (whitespace? ch)
@@ -355,7 +354,7 @@
                  (read-char rdr)
                  (if (== d -1)
                    (throw (IllegalArgumentException. (str "Invalid digit: " ch)))
-                   (recur (inc i) (long (* uc (+ base d))))))))
+                   (recur (inc i) (long (+ d (* uc base))))))))
            (char uc))))))

 (let [upper-limit (int \uD799)
@@ -483,7 +482,8 @@
          ch (read-char reader)]
     (case ch
       nil (reader-error reader "EOF while reading string")
-      \\ (recur (doto sb (.append (escape-char sb reader))) (read-char reader))
+      \\ (recur (doto sb (.append (escape-char sb reader)))
+                (read-char reader))
       \" (str sb)
       (recur (doto sb (.append ch)) (read-char reader)))))


dosync alter in various places

dosync alter in various places

git-svn-id: 03a8048b5ee8463be5048a3801110fb50f378627@6061 d0ab736e-dc22-4aeb-8dc9-08def0aa14fd

--- acqEngine/src/org/micromanager/acq_engine.clj
+++ acqEngine/src/org/micromanager/acq_engine.clj
@@ -228,7 +228,7 @@
     (.put out-queue (annotate-image image event))))

 (defn store-z-correction [z event]
-  (alter state assoc-in [:z-corrections (get-in event [:position :label])] z))
+  (dosync alter state assoc-in [:z-corrections (get-in event [:position :label])] z))

 (defn compute-z-position [event]
   (if-let [z-drive (:z-drive event)]
@@ -261,10 +261,10 @@
     (event-fn)))

 (defn stop-acq []
-  (dosync (alter state assoc-in [:interrupt-requests :stop] true)))
+  (dosync alter state assoc-in [:interrupt-requests :stop] true))

 (defn pause-acq []
-  (dosync (alter state assoc-in [:interrupt-requests :pause] true)))
+  (dosync alter state assoc-in [:interrupt-requests :pause] true))

 (defn run-acquisition [settings out-queue]
   (def acq-settings settings)


multicamera mda

multicamera mda

git-svn-id: 03a8048b5ee8463be5048a3801110fb50f378627@8079 d0ab736e-dc22-4aeb-8dc9-08def0aa14fd

--- acqEngine/src/org/micromanager/acq_engine.clj
+++ acqEngine/src/org/micromanager/acq_engine.clj
@@ -24,7 +24,7 @@
            get-msp MultiStagePosition-to-map ChannelSpec-to-map
            get-pixel-type get-current-time-str rekey
            data-object-to-map str-vector double-vector
-           get-property-value edt]]
+           get-property-value]]
         [org.micromanager.sequence-generator :only [generate-acq-sequence
                                                     make-property-sequences]])
   (:require [clojure.set])
@@ -276,13 +276,6 @@
         tags (parse-core-metadata md)]
     {:pix pix :tags (dissoc tags "StartTime-ms")}))

-(defn make-multicamera-events [event]
-  (let [num-camera-channels (long (core getNumberOfCameraChannels))]
-    (for [camera-channel (range num-camera-channels)]
-      (let [super-channel-index (+ camera-channel (* num-camera-channels (event :channel-index)))]
-        (assoc event :channel-index super-channel-index
-                     :camera-channel-index camera-channel)))))
-
 (defn collect-burst-images [event out-queue]
   (when (first-trigger-missing?) (pop-burst-image)) ; drop first image if first trigger doesn't happen
   (swap! state assoc :burst-time-offset nil)
@@ -299,25 +292,26 @@
                 (swap! state assoc
                        :burst-time-offset (- (elapsed-time @state)
                                              (core-time-from-tags (image :tags)))))
-              ;(println (image+ :tags))
+             ; (println (image+ :tags))
               (.put out-queue (make-TaggedImage (annotate-image image+ evt @state
                                                                 (burst-time (:tags image) @state)))))
             (when (core isBufferOverflowed)
               (swap! state assoc :stop true)
               (ReportingUtils/showError "Circular buffer overflowed."))))
-        (flatten (map make-multicamera-events (event :burst-data))))))
+        (flatten (map make-multicamera-events) (event :burst-data)))))
   (while (and (not (@state :stop)) (. mmc isSequenceRunning))
     (Thread/sleep 5)))

 (defn collect-snap-image [event out-queue]
   (let [image
-        {:pix (core getImage)
-         :tags nil}]
-    (select-keys event [:position-index :frame-index
-                        :slice-index :channel-index])
-    (when out-queue
-      (.put out-queue
-            (make-TaggedImage (annotate-image image event @state (elapsed-time @state)))))
+         {:pix (core getImage)
+          :tags nil}]
+    (do (log "collect-snap-image: "
+               (select-keys event [:position-index :frame-index
+                                   :slice-index :channel-index]))
+      (when out-queue
+          (.put out-queue
+                (make-TaggedImage (annotate-image image event @state (elapsed-time @state))))))
     image))

 (defn return-config []
@@ -372,9 +366,17 @@
                          (:relative-z event))
       nil)))

+(defn make-multicamera-events [event]
+  (let [num-camera-channels (core getNumberOfCameraChannels)]
+    (for [camera-channel (range num-camera-channels)]
+      (let [super-channel-index (+ camera-channel (* num-camera-channels (event :channel-index)))]
+        (assoc event :channel-index super-channel-index
+                     :camera-channel-index camera-channel)))))
+
 (defn collect [event out-queue]
   (condp = (:task event)
-                :snap (collect-snap-image event out-queue)
+                :snap (doseq [sub-event (make-multicamera-events event)]
+                        (collect-snap-image sub-event out-queue))
                 :burst (collect-burst-images event out-queue)))

 (defn z-in-msp [msp z-drive]
@@ -391,21 +393,12 @@
               0))
          z-ref))))

-(defn recall-z-reference [current-position]
+(defn store-new-z-reference [msp]
   (let [z-drive (@state :default-z-drive)]
     (when (and (or (not (core isContinuousFocusEnabled))
-                   (core isContinuousFocusDrive z-drive)))
-      (set-z-stage-position z-drive
-        (or (get-msp-z-position current-position z-drive)
-            (@state :reference-z))))))
-
-(defn store-z-reference [current-position]
-  (let [z-drive (@state :default-z-drive)]
-    (when (and (or (not (core isContinuousFocusEnabled))
-                   (core isContinuousFocusDrive z-drive)))
-      (let [z (get-z-stage-position z-drive)]
-        (set-msp-z-position current-position z-drive z)
-        (state-assoc! :reference-z z)))))
+                   (core isContinuousFocusDrive z-drive))
+               (not (z-in-msp msp z-drive)))
+      (state-assoc! :reference-z (get-z-stage-position z-drive)))))

 ;; startup and shutdown

@@ -441,6 +434,7 @@
       )))

 (defn cleanup []
+  (try
   (log "cleanup")
  ; (do-when #(.update %) (:display @state))
   (state-assoc! :finished true :display nil)
@@ -453,7 +447,8 @@
   (when (and (@state :init-continuous-focus)
              (not (core isContinuousFocusEnabled)))
     (core enableContinuousFocus true))
-  (return-config))
+  (return-config)
+    (catch Throwable t (ReportingUtils/showError t "Acquisition cleanup failed."))))

 ;; running events

@@ -461,6 +456,7 @@
   (let [current-position (:position event)
         z-drive (@state :default-z-drive)
         check-z-ref (and z-drive
+                         (-> current-position get-msp (z-in-msp z-drive) not)
                          (or (:autofocus event)
                              (:wait-time-ms event)))]
     (filter identity
@@ -476,13 +472,13 @@
                        camera (core getCameraDevice)]
              (device-best-effort camera (core setExposure exposure)))
           #(when check-z-ref
-             (recall-z-reference current-position))
+             (set-stage-position z-drive (@state :reference-z)))
           #(when-let [wait-time-ms (:wait-time-ms event)]
              (acq-sleep wait-time-ms))
-          #(when (get event :autofocus)
+          #(when (:autofocus event)
              (run-autofocus))
           #(when check-z-ref
-             (store-z-reference current-position))
+             (store-new-z-reference (get-msp current-position)))
           #(when z-drive
              (let [z (compute-z-position event)]
                (set-stage-position z-drive z)))
@@ -499,6 +495,7 @@

 (defn run-acquisition [this settings out-queue]
   (try
+    (log "Multi-Dimensional Acquisition started.")
     (def acq-settings settings)
     (prepare-state this)
     (binding [state (.state this)]
@@ -515,7 +512,6 @@
 ;; generic metadata

 (defn convert-settings [^SequenceSettings settings]
-  (def seqSettings settings)
   (-> settings
     (data-object-to-map)
     (rekey
@@ -553,15 +549,16 @@
                 (:properties channel)))
             default-cam)]
     (set-property chan-cam)
-    (let [n (long (core getNumberOfComponents))]
+    (let [n (core getNumberOfComponents)]
       (set-property default-cam)
       (get {1 1 , 4 3} n))))

 (defn make-summary-metadata [settings]
-  (let [depth (core getBytesPerPixel)
+  (let [depth (int (core getBytesPerPixel))
         channels (settings :channels)
-        num-camera-channels (long (core getNumberOfCameraChannels))
-        super-channels (flatten (for [channel channels] (repeat num-camera-channels channel)))]
+        num-camera-channels (core getNumberOfCameraChannels)
+        simple-channels (if-not (empty? channels) channels [{:name "Default" :color java.awt.Color/WHITE}])
+        super-channels (flatten (for [channel simple-channels] (repeat num-camera-channels channel)))]
      (JSONObject. {
       "BitDepth" (core getImageBitDepth)
       "Channels" (count super-channels)
@@ -616,11 +613,6 @@
     image-tags
     ["Width" "Height" "PixelType"]))

-(defn initialize-display-ranges [window]
-  (do (.setChannelDisplayRange window 0 0 256)
-      (.setChannelDisplayRange window 1 0 256)
-      (.setChannelDisplayRange window 2 0 256)))
-
 (defn create-image-window [first-image]
   (let [summary {:interval-ms 0.0, :custom-intervals-ms [] :use-autofocus false, :autofocus-skip 0,
                  :relative-slices true, :keep-shutter-open-slices false, :comment "",
@@ -630,10 +622,8 @@
                  :zReference 0.0, :frames (), :save false}
 		summary-metadata (make-summary-metadata summary)
 		cache (doto (MMImageCache. (TaggedImageStorageRam. summary-metadata))
-						(.setSummaryMetadata summary-metadata))]
-		(doto (VirtualAcquisitionDisplay. cache nil)
-                           (.promptToSave false)
-                           (initialize-display-ranges))))
+						(.setSummaryMetadata summary-metadata))]
+    (doto (VirtualAcquisitionDisplay. cache nil))))

 (defn create-basic-event []
   {:position-index 0, :position nil,
@@ -658,7 +648,7 @@
   (let [myTaggedImage (make-TaggedImage tagged-img)
         cache (.getImageCache display)]
     (.putImage cache myTaggedImage)
-    (.showImage display (. myTaggedImage tags) true false)
+    (.showImage display myTaggedImage)
     (when focus
       (.show display))))

@@ -682,31 +672,25 @@
 (def live-mode-running (ref false))

 (defn enable-live-mode [^Boolean on]
-  (if on
-    (let [event (create-basic-event)
-          state (create-basic-state)
-          start-time (jvm-time-ms)]
-      (dosync (ref-set live-mode-running true))
-      (core startContinuousSequenceAcquisition 0)
-      (log "started sequence acquisition")
-      (.start (Thread.
-                #(dorun (loop [last-time 0
-                               first-image true]
-                          (when @live-mode-running
-                            (let [raw-image {:pix (core getLastImage) :tags nil}
-                                  elapsed-time-ms (- (jvm-time-ms) start-time)
-                                  img (annotate-image raw-image event state elapsed-time-ms)
-                                  this-time (System/currentTimeMillis)
-                                  dt (- 33 (- this-time last-time))]
-                              (if first-image (reset-snap-window img)  )
-                              (Thread/sleep (max 0 dt))
-                              (if (.windowClosed @snap-window)
-                                (edt (.enableLiveMode gui false))
-                                (show-image @snap-window img first-image)  )
-                              (recur this-time false))))
-                        (core stopSequenceAcquisition))
-                "Live mode (clojure)")))
-    (dosync (ref-set live-mode-running false))))
+      (if on
+        (let [event (create-basic-event)
+              state (create-basic-state)
+              first-image (atom true)
+              start-time (jvm-time-ms)]
+          (dosync (ref-set live-mode-running true))
+          (core startContinuousSequenceAcquisition 0)
+          (log "started sequence acquisition")
+          (.start (Thread.
+                    #(do (while @live-mode-running
+                           (let [raw-image {:pix (core getLastImage) :tags nil}
+                                 elapsed-time-ms (- (jvm-time-ms) start-time)
+                                 img (annotate-image raw-image event state elapsed-time-ms)]
+                              (reset-snap-window img)
+                              (show-image @snap-window img @first-image)
+                              (reset! first-image false)))
+                         (core stopSequenceAcquisition))
+                    "Live mode (clojure)")))
+        (dosync (ref-set live-mode-running false))))

 ;; java interop


Use a port that's easier to type for connecting Swank to a testing environment.

--- src/circle/swank.clj
+++ src/circle/swank.clj
@@ -4,7 +4,7 @@

 (defn port []
   (cond
-      (env/test?) 5004
+      (env/test?) 4006
       :else 4005))

 (defn init []


blink before collapsing

blink before collapsing

--- src/client/core.cljs
+++ src/client/core.cljs
@@ -30,7 +30,8 @@
 ;; Board.
 ;;------------------------------------------------------------

-(def empty-row (vec (repeat 10 0)))
+(def empty-row       (vec (repeat 10 0)))
+(def highlighted-row (vec (repeat 10 :H)))
 (def empty-board (vec (repeat 22 empty-row)))

 ; The starting position of all pieces.
@@ -54,7 +55,9 @@
    :Z "#F00"
    :O "#FF0"
    :T "#A0F"
-   :G "#555"})
+   :G "#555"  ; ghost piece
+   :H "#DDD"  ; highlighted (filled or about to collapse)
+   })

 (def cell-filled? (complement zero?))

@@ -96,6 +99,21 @@
         coords (:coords piece)]
     (write-coords-to-board coords x y value board)))

+(defn highlight-rows
+  "Returns a new board with the given rows highlighted."
+  [active-rows board]
+  (vec (map-indexed
+   (fn [i row]
+     (if (active-rows i) highlighted-row row)) board)))
+
+(defn get-filled-row-indices
+  "Get the indices of the filled rows for the given board."
+  [board]
+  (->> (map-indexed vector board)                        ; indexed rows [[0 r] [1 r]]
+       (filter (fn [[i row]] (every? cell-filled? row))) ; choose filled [1 r]
+       (map first)                                       ; select index only
+       (apply hash-set)))                                ; convert to a set
+
 (defn rotate-piece
   "Create a new piece by rotating the given piece clockwise."
   [piece]
@@ -176,7 +194,10 @@

 (def state (atom {:piece (get-rand-piece)
                   :position start-position
-                  :board empty-board}))
+                  :board empty-board
+
+                  :animating false
+                  :highlighted-rows #{}}))

 ;;------------------------------------------------------------
 ;; VCR (record game)
@@ -248,8 +269,9 @@
         ghost (assoc piece :name :G)
         gy    (get-drop-pos piece x y board)
         board1 (write-piece-to-board ghost x gy board)
-        board2 (write-piece-to-board piece x y board1)]
-    board2))
+        board2 (write-piece-to-board piece x y board1)
+        board3 (highlight-rows (:highlighted-rows @state) board2)]
+    board3))

 (defn draw-state
   "Draw the current state of the board."
@@ -270,8 +292,8 @@
   (swap! state assoc :piece (get-rand-piece)
                      :position start-position))

-(defn try-collapse!
-  "Try to collapse any full rows on the current board."
+(defn collapse-rows!
+  "Collapse all filled rows."
   []
   (let [board (:board @state)
         cleared-board (remove #(every? cell-filled? %) board)
@@ -279,6 +301,30 @@
         new-board (into (vec (repeat n empty-row)) cleared-board)]
     (swap! state assoc :board new-board)))

+(defn go-go-collapse!
+  []
+  (let [board (:board @state)
+        rows (get-filled-row-indices board)]
+
+    (js/console.log "Collapsing?")
+    (when (> (count rows) 0)
+      (js/console.log "YES")
+      (swap! state assoc :animating true)
+      (go
+        ; blink n times
+        (doseq [i (range 3)]
+          (js/console.log "Blink" i)
+
+          (<! (timeout 100))                            ; resume here in 0.5 seconds
+          (swap! state assoc :highlighted-rows rows)    ; highlight rows
+          (<! (timeout 100))                            ; resume here in 0.5 seconds
+          (swap! state assoc :highlighted-rows #{}))     ; unhighlight rows
+
+        ; finally collapse
+        (js/console.log "Collapsing.")
+        (collapse-rows!)
+        (swap! state assoc :animating false)))))
+
 (defn lock-piece!
   "Lock the current piece into the board."
   []
@@ -286,7 +332,7 @@
         piece (:piece @state)
         board (:board @state)]
     (swap! state assoc :board (write-piece-to-board piece x y board))
-    (try-collapse!)
+    (go-go-collapse!)
     (spawn-piece!)))

 (defn go-go-gravity!
@@ -295,13 +341,14 @@
   (go
     (loop []
       (<! (timeout 1000))
-      (let [[x y] (:position @state)
-            piece (:piece @state)
-            board (:board @state)
-            ny (inc y)]
-        (if (piece-fits? piece x ny board)
-          (swap! state assoc-in [:position 1] ny)
-          (lock-piece!)))
+      (when-not (:animating @state)
+        (let [[x y] (:position @state)
+              piece (:piece @state)
+              board (:board @state)
+              ny (inc y)]
+          (if (piece-fits? piece x ny board)
+            (swap! state assoc-in [:position 1] ny)
+            (lock-piece!))))
       (recur))))

 ;;------------------------------------------------------------


updating the chat atom test

updating the chat atom test

--- src/client/game.cljs
+++ src/client/game.cljs
@@ -4,6 +4,7 @@
     [cljs.reader :refer [read-string]]
     hiccups.runtime
     client.game.core
+    client.state
     [client.dom :as dom]
     [client.socket :as socket]
     [client.util :as util]))
@@ -212,6 +213,10 @@
 ;; var objDiv = document.getElementById("your_div");
 ;; objDiv.scrollTop = objDiv.scrollHeight;

+(defn- on-change-chat [_ _ old-chat new-chat]
+  (util/log new-chat)
+  )
+
 ;;------------------------------------------------------------------------------
 ;; DOM Events
 ;;------------------------------------------------------------------------------
@@ -238,7 +243,8 @@
           msg (aget input-el "value")]

       ;; TODO: send chat message here
-      (util/log (str "chat message: " msg))
+      ;;(util/log (str "chat message: " msg))
+      (socket/send-chat msg)

       (aset input-el "value" ""))))

@@ -256,6 +262,7 @@

 ;; TODO: not finished yet!
 (defn- init2 []
+  (add-watch client.state/chat :game-screen on-change-chat)
   (reset! initialized false)
   (dom/set-bw-background!)
   (dom/set-page-body! (page-shell))
@@ -295,6 +302,8 @@
 (defn cleanup
   []

+  (remove-watch client.state/chat :game-screen)
+
   ; Leave the game room.
   (socket/emit "leave-game")


Remove throw exception on missing env

Remove throw exception on missing env

--- src/clj/inky/env.clj
+++ src/clj/inky/env.clj
@@ -15,8 +15,7 @@
   exception."
   [sym & [default]]
   (or (System/getenv (clj->env sym))
-      default
-      (throw (Exception. (clojure.core/str "Missing env var " (clj->env sym) ".")))))
+      default))

 (defn int
   "Retrieve and parse int env var."


Extract fn

Extract fn

--- src/clojure/core.clj
+++ src/clojure/core.clj
@@ -354,6 +354,9 @@
        acc
        fields))))

+(defn max-for [field frame-stats]
+  (apply max (map (fn [f] (get (first (vals f)) field))  frame-stats)))
+
 (defn global-stats [frame-stats]
   (let [n (count frame-stats)
         fields (-> frame-stats first vals first (dissoc :todo) keys)
@@ -363,11 +366,7 @@
                                         (= (:f0 stats) 0.0)
                                        )) frame-stats)
         n-f0s (count f0-frames)
-        f0-totals (reduce (global-stats-reduce fields) {} f0-frames)
-
-
-        max-fn (fn [field] (apply max (map (fn [f] (get (first (vals f)) field ))  frame-stats)) )
-        ]
+        f0-totals (reduce (global-stats-reduce fields) {} f0-frames)]

     (let [means
           (-> {}
@@ -382,8 +381,7 @@
               (assoc :midi (/ (:midi f0-totals) n-f0s))
               (assoc :f0   (/ (:f0 f0-totals) n-f0s)))

-          maxs {:spectral-centroid (max-fn :spectral-centroid)}
-          ]
+          maxs {:spectral-centroid (max-for :spectral-centroid frame-stats)}]
       {:mean means
        :max maxs}
   )))
@@ -391,9 +389,7 @@
 (comment

   (def all-wavs (filter #(.endsWith (.getName %) ".wav") (file-seq (io/file "/Users/josephwilk/Workspace/music/samples"))))
-
   (doseq [w (take 1000 all-wavs)]
-
     (try
       (let [wav (.getPath w) ;; "test/fixtures/test.wav" ;;(.getPath wav)
             _ (println wav)
@@ -405,6 +401,7 @@

       (catch Exception e true)
       ))
+
   (dotimes [i 1]
     (let [block-stats (block-stats "test/fixtures/test.wav")]
 ;;      (doseq [frame block-stats] (println frame))


Changed the default :age-combining-function to average.

--- src/clojush/args.clj
+++ src/clojush/args.clj
@@ -225,7 +225,7 @@
           ;; :atom-generators for autoconstruction. If negative then autoconstructive_boolean_rand
           ;; will not be in :atom-generators at all.

-          :age-combining-function max
+          :age-combining-function average
           ;; For genetic operators that involve multiple parents, the function used to combine
           ;; the incremented ages of the parents to produce the age of the child.

@@ -564,3 +564,4 @@

+


[#160] Use an existing namespace for the test

--- priv/clojure/test_clojure/reader.clj
+++ priv/clojure/test_clojure/reader.clj
@@ -279,10 +279,10 @@
   )

 (deftest reading-keywords
-  (are [x y] (= x (binding [*ns* (the-ns 'user)] (read-string y)))
+  (are [x y] (= x (binding [*ns* (the-ns 'clje.user)] (read-string y)))
        :foo ":foo"
        :foo/bar ":foo/bar"
-       :user/foo "::foo")
+       :clje.user/foo "::foo")
   (are [err msg form] (thrown-with-msg? err msg (read-string form))
        :error #"Invalid token: foo:" "foo:"
        :error #"Invalid token: :bar/" ":bar/"


Touch up tests for zrange and zrangebyscore

--- src/test/clojure/tests.clj
+++ src/test/clojure/tests.clj
@@ -378,13 +378,19 @@
   (redis/zadd "zset" 12349809.23873948579348750 "one")
   (redis/zadd "zset" -42 "two")
   (redis/zadd "zset" 3.141592 "three")
+  (is (= [] (redis/zrange "zset" -1 -2)))
   (is (= ["two" "three" "one"] (redis/zrange "zset" 0 2)))
   (is (= ["three" "one"] (redis/zrange "zset" 1 2))))

 (deftest zrangebyscore
   (is (thrown? Exception (redis/zrangebyscore "foo")))
-  (is (= nil (redis/zrangebyscore "zset" 0 99))))
-
+  (is (= nil (redis/zrangebyscore "zset" 0 99)))
+  (redis/zadd "zset" 1.0 "one")
+  (redis/zadd "zset" 2.0 "two")
+  (redis/zadd "zset" 3.0 "three")
+  (is (= [] (redis/zrangebyscore "zset" -42 0.99)))
+  (is (= ["two"] (redis/zrangebyscore "zset" 1.1 2.9)))
+  (is (= ["two" "three"] (redis/zrangebyscore "zset" 1.0000001 3.00001))))

 ;;
 ;; Sorting


Stop chrome adding `<div>`s when pressing enter in contentEditables

--- src/cljs/gc_api_browser/tabbed_request.cljs
+++ src/cljs/gc_api_browser/tabbed_request.cljs
@@ -2,15 +2,23 @@
   (:require [om.core :as om :include-macros true]
             [om.dom :as dom :include-macros true]
             [gc-api-browser.headers :as headers]))
+
+(def ENTER 13)
+
+(defn handle-key-down [e]
+  (when (= (.-keyCode e) ENTER)
+    (.execCommand js/document "insertHTML" false "\n")
+    (.preventDefault e)))

 (defn content-editable [cursor owner]
   (reify om/IRender
     (render [_]
       (dom/code #js {:contentEditable true
                      :dangerouslySetInnerHTML #js {:__html (:body cursor)}
+                     :onKeyDown handle-key-down
                      :onInput (fn [_]
                                 (let [html (.-innerHTML (om/get-node owner))]
-                                  (om/update! cursor :body html)))}))))
+                                  (om/transact! cursor :body (fn [_] html))))}))))

 (defn edit-body [{:keys [body] :as cursor}]
   (dom/pre #js {:className "flex-container tabbed-request__body"}


cleanup login page

cleanup login page

--- src/cljs/swarmpit/component/page_login.cljs
+++ src/cljs/swarmpit/component/page_login.cljs
@@ -132,8 +132,7 @@
                               :message      ""
                               :canSubmit    true
                               :showPassword false} ::login)
-                  rum/reactive
-                  menu/retrieve-version [state]
+                  rum/reactive [state]
   (let [local-state (::login state)
         initialized (state/react [:initialized])
         username (:username @local-state)


Removed extra ?pix-idx value from timeseries generation.

--- src/clj/forma/hadoop/jobs/load_tseries.clj
+++ src/clj/forma/hadoop/jobs/load_tseries.clj
@@ -30,7 +30,7 @@
   "Given a source of chunks, this subquery generates timeseries with
   all relevant accompanying information."
   [chunk-source]
-  (<- [?dataset ?spatial-res ?temporal-res ?tile-h ?tile-v ?sample ?line ?pix-idx ?t-start ?t-end ?tseries]
+  (<- [?dataset ?spatial-res ?temporal-res ?tile-h ?tile-v ?sample ?line ?t-start ?t-end ?tseries]
       (chunk-source ?dataset ?spatial-res ?temporal-res ?tilestring ?date ?chunkid ?int-chunk)
       (count ?int-chunk :> ?chunk-size)
       (extract-tseries ?temporal-res ?date ?int-chunk :> ?pix-idx ?t-start ?t-end ?tseries)


Improve distribution of generated weight matrices

--- src/cortex/util.cljc
+++ src/cortex/util.cljc
@@ -79,8 +79,12 @@
   (m/emul y (m/sub 1.0 y)))

 (defn weight-matrix
+  "Creates a randomised weight matrix.
+
+   Weights are gaussian values scaled acoording to 1/sqrt(no. columns). This ensures that outputs are distributed
+   on a similar scale to inputs, and provides reasonable initial gradient propagation."
   [rows cols]
-  (let [weight-scale (Math/sqrt (/ 1.0 (* (double rows) (double cols))))]
+  (let [weight-scale (Math/sqrt (/ 1.0 (double cols)))]
     (b/array
      (mapv (fn [_]
              (vec (repeatedly cols


Conditionally define the ::fulcro.network/progress-handler

--- src/com/wsscode/pathom/fulcro/network.cljs
+++ src/com/wsscode/pathom/fulcro/network.cljs
@@ -101,21 +101,18 @@
                    transform-progress (fn [_ x] x)}} options
           req-id (random-uuid)
           env    {::request-id req-id
-                  ::app        @app*}]
+                  ::app        @app*}
+          options (cond-> {::fulcro.network/ok-handler    (fn [response] (ok-handler (update response :body #(transform-response env %))))
+                           ::fulcro.network/error-handler (fn [error] (error-handler (update error :body #(transform-error env %))))}
+
+                    progress-handler
+                    (assoc ::fulcro.network/progress-handler (fn [progress] (progress-handler (transform-progress env progress)))))]
       (if-let [edn' (transform-query env edn)]
         (if transform-transmission
           (transform-transmission env edn'
             (fn [edn']
-              (fulcro.network/transmit network
-                {::fulcro.network/edn              edn'
-                 ::fulcro.network/ok-handler       (fn [response] (ok-handler (update response :body #(transform-response env %))))
-                 ::fulcro.network/error-handler    (fn [error] (error-handler (update error :body #(transform-error env %))))
-                 ::fulcro.network/progress-handler (fn [progress] (progress-handler (transform-progress env progress)))})))
-          (fulcro.network/transmit network
-            {::fulcro.network/edn              edn'
-             ::fulcro.network/ok-handler       (fn [response] (ok-handler (update response :body #(transform-response env %))))
-             ::fulcro.network/error-handler    (fn [error] (error-handler (update error :body #(transform-error env %))))
-             ::fulcro.network/progress-handler (fn [progress] (progress-handler (transform-progress env progress)))}))
+              (fulcro.network/transmit network (assoc options ::fulcro.network/edn edn'))))
+          (fulcro.network/transmit network (assoc options ::fulcro.network/edn edn')))
         (ok-handler nil))))

   (abort [this abort-id]


Refactoring: web handlers

Refactoring: web handlers

--- src/export_server/handlers/web_handlers.clj
+++ src/export_server/handlers/web_handlers.clj
@@ -13,8 +13,7 @@
             [export-server.web.params-validator :as params-validator]

             [export-server.utils.rasterizator :as rastr]
-            [export-server.utils.rasterizator :as rast]
-    ;[export-server.browser.core :as browser]
+   ;[export-server.browser.core :as browser]
             [export-server.browser.raw :as browser]
             [export-server.sharing.twitter :as twitter]
             [me.raynes.fs :as fs])
@@ -158,7 +157,7 @@

 (defn save-file-and-get-url [data file-name extention]
   (if-let [folder (:saving-folder @state/options)]
-    (let [new-file-name (rast/get-file-name-hash file-name)
+    (let [new-file-name (rastr/get-file-name-hash file-name)
           path (str folder "/" new-file-name extention)]
       (fs/mkdirs folder)
       (io/copy data (io/file path))
@@ -174,7 +173,7 @@
     (if (params-validator/valid-result? validation-result)
       (let [{ok :ok result :result} (to-png params)]
         (if ok
-          (twitter/twitter request (rast/to-base64 result))
+          (twitter/twitter request (rastr/to-base64 result))
           (log/wrap-log-error json-error result request :processing)))
       (log/wrap-log-error json-error (params-validator/get-error-message validation-result) request :bad_params))))

@@ -187,8 +186,8 @@
         (if ok
           (if (= response-type "base64")
             (if (params "save")
-              (save-file-and-get-url (rast/to-base64 result) (get-file-name params) ".base64")
-              (json-success (rast/to-base64 result)))
+              (save-file-and-get-url (rastr/to-base64 result) (get-file-name params) ".base64")
+              (json-success (rastr/to-base64 result)))
             (if (params "save")
               (save-file-and-get-url result (get-file-name params) ".png")
               (file-success result (get-file-name params) ".png")))
@@ -205,8 +204,8 @@
         (if ok
           (if (= response-type "base64")
             (if (params "save")
-              (save-file-and-get-url (rast/to-base64 result) (get-file-name params) ".base64")
-              (json-success (rast/to-base64 result)))
+              (save-file-and-get-url (rastr/to-base64 result) (get-file-name params) ".base64")
+              (json-success (rastr/to-base64 result)))
             (if (params "save")
               (save-file-and-get-url result (get-file-name params) ".jpg")
               (file-success result (get-file-name params) ".jpg")))
@@ -223,8 +222,8 @@
         (if ok
           (if (= response-type "base64")
             (if (params "save")
-              (save-file-and-get-url (rast/to-base64 result) (get-file-name params) ".base64")
-              (json-success (rast/to-base64 result)))
+              (save-file-and-get-url (rastr/to-base64 result) (get-file-name params) ".base64")
+              (json-success (rastr/to-base64 result)))
             (if (params "save")
               (save-file-and-get-url result (get-file-name params) ".pdf")
               (file-success result (get-file-name params) ".pdf")))
@@ -241,8 +240,8 @@
         (if ok
           (if (= response-type "base64")
             (if (params "save")
-              (save-file-and-get-url (rast/to-base64 (.getBytes result)) (get-file-name params) ".base64")
-              (json-success (rast/to-base64 (.getBytes result))))
+              (save-file-and-get-url (rastr/to-base64 (.getBytes result)) (get-file-name params) ".base64")
+              (json-success (rastr/to-base64 (.getBytes result))))
             (if (params "save")
               (save-file-and-get-url (.getBytes result) (get-file-name params) ".svg")
               (file-success (.getBytes result) (get-file-name params) ".svg")))


for French, add gender symbols before generation of surface for "they" and "it"

--- src/babel/english/morphology.cljc
+++ src/babel/english/morphology.cljc
@@ -24,7 +24,33 @@
                 ;; remove :notes in these circumstances.
                 (dissoc-paths input [[:english :a :note] ;; this handles "you" in "you wash yourself"
                                      [:english :b :b :note]]) ;; this handles "yourself" in the same sentence.
-                input)]
+                input)
+
+        input (cond (and (= "fr" from-language)
+                         (= "they" (get-in input [:english :a :english]))
+                         (= :fem (get-in input [:english :a :agr :gender])))
+                    (assoc-in
+                     input [:english :a :note] "♀")
+                    (and (= "fr" from-language)
+                         (= "they" (get-in input [:english :a :english]))
+                         (= :masc (get-in input [:english :a :agr :gender])))
+                    (assoc-in
+                     input [:english :a :note] "♂")
+
+                    (and (= "fr" from-language)
+                         (= "it" (get-in input [:english :a :english]))
+                         (= :fem (get-in input [:english :a :agr :gender])))
+                    (assoc-in
+                     input [:english :a :note] "♀")
+                    (and (= "fr" from-language)
+                         (= "it" (get-in input [:english :a :english]))
+                         (= :masc (get-in input [:english :a :agr :gender])))
+                    (assoc-in
+                     input [:english :a :note] "♂")
+
+                    true input)]
+
+
     (cond ;; fo
       (= input :fail)
       (str input)


+clojure.repl/doc

+clojure.repl/doc

--- test/babel/test/it.cljc
+++ test/babel/test/it.cljc
@@ -11,6 +11,7 @@
             #?(:cljs [cljs.test :refer-macros [deftest is]])
             #?(:clj [clojure.tools.logging :as log])
             #?(:cljs [babel.logjs :as log])
+            [clojure.repl :refer [doc]]
             [clojure.string :as string]
             [clojure.set :as set]
             [dag_unify.core :refer [copy get-in strip-refs]]))


use "ardere" for test

use "ardere" for test

--- test/babel/test/la.cljc
+++ test/babel/test/la.cljc
@@ -10,7 +10,7 @@
 ;; https://en.wikipedia.org/wiki/Latin_conjugation#Present_indicative
 (deftest analyze-ere
   (is (= :verb
-         (-> (analyze "abeo" lexicon)
+         (-> (analyze "ardeo" lexicon)
              first
              (get-in [:synsem :cat])))))


add tests for imperfect and future (as well as present)

--- test/babel/test/la.cljc
+++ test/babel/test/la.cljc
@@ -21,16 +21,32 @@
          (conjugate "ardēre"
                     {:synsem {:sem {:subj {:pred :noi}}}}))))

-(deftest generate-1
+(deftest generate-present
   (is (= "ardetis"
          (fo (generate
               {:root "ardēre"
-               :synsem {:sem {:subj {:pred :voi}}}})))))
+               :synsem {:sem {:subj {:pred :voi}
+                              :tense :present}}})))))

-
+(deftest generate-imperfect
+  (is (= "ardebam"
+         (fo (generate
+              {:root "ardēre"
+               :synsem {:sem {:subj {:pred :I}
+                              :tense :past
+                              :aspect :progressive}}})))))
+(deftest generate-future
+  (is (= "ardebunt"
+         (fo (generate
+              {:root "ardēre"
+               :synsem {:sem {:subj {:pred :loro}
+                              :tense :future}}})))))
 (deftest engine-generate
   (is (= "ardetis"
          (fo (engine/generate
               {:root "ardēre"
-               :synsem {:sem {:subj {:pred :voi}}}}
+               :synsem {:sem {:subj {:pred :voi}
+                              :tense :present}}}
               model)))))
+
+


Add fastmap tests

Add fastmap tests

--- test/clj_bwa/t_core.clj
+++ test/clj_bwa/t_core.clj
@@ -4,7 +4,7 @@
             [clojure.java.io :as io]
             [me.raynes.fs :as fs]
             [clj-bwa.core :as bwa])
-  (:import [clj_bwa.jna MemOption]))
+  (:import [clj_bwa.jna MemOption FastmapOption]))

 (defmacro with-out-file
   [f & body]
@@ -124,7 +124,7 @@
               (bwa/sw-option))) => anything))

 ;; mem test
-;; ----------
+;; --------

 (fact "about mem-option"
   (type (bwa/mem-option)) => MemOption
@@ -144,3 +144,24 @@
                nil
                (str temp-dir "/out.sam")
                (bwa/mem-option))) => anything))
+
+;; fastmap test
+;; ------------
+
+(fact "about fastmap-option"
+  (type (bwa/fastmap-option)) => FastmapOption
+  (type (bwa/fastmap-option nil)) => FastmapOption
+  (type (bwa/fastmap-option {})) => FastmapOption
+  (type (bwa/fastmap-option {:min-len 100})) => FastmapOption)
+
+(with-state-changes [(before :facts (do (prepare-cache!)
+                                        (doseq [f test-db-files]
+                                          (fs/copy f (str temp-dir "/" (fs/base-name f))))
+                                        (fs/copy test-fq-file (str temp-dir "/test.fq"))))
+                     (after :facts (clean-cache!))]
+  (fact "about fastmap"
+    (with-out-file temp-out
+      (bwa/fastmap (str temp-dir "/test.fa")
+                   (str temp-dir "/test.fq")
+                   (str temp-dir "/out.fastmap")
+                   (bwa/fastmap-option))) => anything))


refine intersection

refine intersection

--- test/cljam/t_sorter.clj
+++ test/cljam/t_sorter.clj
@@ -5,64 +5,69 @@
             [cljam.bam :as bam]
             [cljam.sorter :as sorter]))

-(def tmp-shuffled-sam-file (str temp-dir "/" "tmp.shuffled.sam"))
-(def tmp-shuffled-bam-file (str temp-dir "/" "tmp.shuffled.bam"))
-
 (def tmp-coordinate-sorted-sam-file (str temp-dir "/" "tmp.coordinate.sorted.sam"))
 (def tmp-coordinate-sorted-bam-file (str temp-dir "/" "tmp.coordinate.sorted.bam"))

 (def tmp-queryname-sorted-sam-file (str temp-dir "/" "tmp.queryname.sorted.sam"))
 (def tmp-queryname-sorted-bam-file (str temp-dir "/" "tmp.queryname.sorted.bam"))

-(defn prepare-shuffled-files!
+(def tmp-shuffled-sam-file (str temp-dir "/" "tmp.shuffled.sam"))
+(def tmp-shuffled-bam-file (str temp-dir "/" "tmp.shuffled.bam"))
+
+(defn- prepare-shuffled-files!
   []
   (sam/spit tmp-shuffled-sam-file (get-shuffled-test-sam))
   (bam/spit tmp-shuffled-bam-file (get-shuffled-test-sam)))
+
+(defn- with-reader
+  [target-fn src-file & [dst-file]]
+  (cond
+    (re-find #"\.sam$" src-file) (if dst-file
+                                   (target-fn (sam/reader src-file)
+                                              (sam/writer dst-file))
+                                   (target-fn (sam/reader src-file)))
+    (re-find #"\.bam$" src-file) (if dst-file
+                                   (target-fn (bam/reader src-file)
+                                              (bam/writer dst-file))
+                                   (target-fn (bam/reader src-file)))
+    :else (throw (RuntimeException. (str "invalid file suffix " src-file)))))

 (with-state-changes [(before :facts (do (prepare-cache!)
                                         (prepare-shuffled-files!)))
                      (after  :facts (clean-cache!))]
   (fact "about sorting a sam by chromosomal positions"
         ;; tests by test-sam-file and test-bam-file
-        (sorter/sort-by-pos (sam/reader test-sam-file)
-                            (sam/writer tmp-coordinate-sorted-sam-file)) => anything
-        (sorter/sort-by-pos (bam/reader test-bam-file)
-                            (bam/writer tmp-coordinate-sorted-bam-file)) => anything
-        (sorter/sort-by-qname (sam/reader test-sam-file)
-                              (sam/writer tmp-queryname-sorted-sam-file)) =future=> anything
-        (sorter/sort-by-qname (bam/reader test-bam-file)
-                              (bam/writer tmp-queryname-sorted-bam-file)) =future=> anything
-        (sorter/sorted-by? (sam/reader test-sam-file)) => falsey
-        (sorter/sorted-by? (bam/reader test-bam-file)) => falsey
-        (sorter/sorted-by? (sam/reader tmp-coordinate-sorted-sam-file)) => truthy
-        (sorter/sorted-by? (bam/reader tmp-coordinate-sorted-bam-file)) => truthy
-        (sorter/sorted-by? (sam/reader tmp-queryname-sorted-sam-file)) =future=> truthy
-        (sorter/sorted-by? (bam/reader tmp-queryname-sorted-bam-file)) =future=> truthy
-        (sorter/sort-order (sam/reader test-sam-file)) => sorter/order-unknown
-        (sorter/sort-order (bam/reader test-bam-file)) => sorter/order-unknown
-        (sorter/sort-order (sam/reader tmp-coordinate-sorted-sam-file)) => sorter/order-coordinate
-        (sorter/sort-order (bam/reader tmp-coordinate-sorted-bam-file)) => sorter/order-coordinate
-        (sorter/sort-order (sam/reader tmp-queryname-sorted-sam-file)) =future=> sorter/order-queryname
-        (sorter/sort-order (bam/reader tmp-queryname-sorted-bam-file)) =future=> sorter/order-queryname
+        (with-reader sorter/sort-by-pos test-sam-file tmp-coordinate-sorted-sam-file) => anything
+        (with-reader sorter/sort-by-pos test-bam-file tmp-coordinate-sorted-bam-file) => anything
+        (with-reader sorter/sort-by-qname test-sam-file tmp-coordinate-sorted-sam-file) =future=> anything
+        (with-reader sorter/sort-by-qname test-bam-file tmp-coordinate-sorted-bam-file) =future=> anything
+        (with-reader sorter/sorted-by? test-sam-file) => falsey
+        (with-reader sorter/sorted-by? test-bam-file) => falsey
+        (with-reader sorter/sorted-by? tmp-coordinate-sorted-sam-file) => truthy
+        (with-reader sorter/sorted-by? tmp-coordinate-sorted-bam-file) => truthy
+        (with-reader sorter/sorted-by? tmp-queryname-sorted-sam-file) =future=> truthy
+        (with-reader sorter/sorted-by? tmp-queryname-sorted-bam-file) =future=> truthy
+        (with-reader sorter/sort-order test-sam-file) => sorter/order-unknown
+        (with-reader sorter/sort-order test-bam-file) => sorter/order-unknown
+        (with-reader sorter/sort-order tmp-coordinate-sorted-sam-file) => sorter/order-coordinate
+        (with-reader sorter/sort-order tmp-coordinate-sorted-bam-file) => sorter/order-coordinate
+        (with-reader sorter/sort-order tmp-queryname-sorted-sam-file) =future=> sorter/order-queryname
+        (with-reader sorter/sort-order tmp-queryname-sorted-bam-file) =future=> sorter/order-queryname
         ;; tests by shuffled files (its may sorted by chance)
-        (sorter/sort-by-pos (sam/reader tmp-shuffled-sam-file)
-                            (sam/writer tmp-coordinate-sorted-sam-file)) => anything
-        (sorter/sort-by-pos (bam/reader tmp-shuffled-bam-file)
-                            (bam/writer tmp-coordinate-sorted-bam-file)) => anything
-        (sorter/sort-by-qname (sam/reader tmp-shuffled-sam-file)
-                              (sam/writer tmp-queryname-sorted-sam-file)) =future=> anything
-        (sorter/sort-by-qname (bam/reader tmp-shuffled-bam-file)
-                              (bam/writer tmp-queryname-sorted-bam-file)) =future=> anything
-        (sorter/sorted-by? (sam/reader tmp-shuffled-sam-file)) => anything
-        (sorter/sorted-by? (bam/reader tmp-shuffled-bam-file)) => anything
-        (sorter/sorted-by? (sam/reader tmp-coordinate-sorted-sam-file)) => truthy
-        (sorter/sorted-by? (bam/reader tmp-coordinate-sorted-bam-file)) => truthy
-        (sorter/sorted-by? (sam/reader tmp-queryname-sorted-sam-file)) =future=> truthy
-        (sorter/sorted-by? (bam/reader tmp-queryname-sorted-bam-file)) =future=> truthy
-        (sorter/sort-order (sam/reader tmp-shuffled-sam-file)) => #(#{:queryname :coordinate :unsorted :unknown} %)
-        (sorter/sort-order (bam/reader tmp-shuffled-bam-file)) => #(#{:queryname :coordinate :unsorted :unknown} %)
-        (sorter/sort-order (sam/reader tmp-coordinate-sorted-sam-file)) => sorter/order-coordinate
-        (sorter/sort-order (bam/reader tmp-coordinate-sorted-bam-file)) => sorter/order-coordinate
-        (sorter/sort-order (sam/reader tmp-queryname-sorted-sam-file)) =future=> sorter/order-queryname
-        (sorter/sort-order (bam/reader tmp-queryname-sorted-bam-file)) =future=> sorter/order-queryname
+        (with-reader sorter/sort-by-pos tmp-shuffled-sam-file tmp-coordinate-sorted-sam-file) => anything
+        (with-reader sorter/sort-by-pos tmp-shuffled-bam-file tmp-coordinate-sorted-bam-file) => anything
+        (with-reader sorter/sort-by-qname tmp-shuffled-sam-file tmp-coordinate-sorted-sam-file) =future=> anything
+        (with-reader sorter/sort-by-qname tmp-shuffled-bam-file tmp-coordinate-sorted-bam-file) =future=> anything
+        (with-reader sorter/sorted-by? tmp-shuffled-sam-file) => anything
+        (with-reader sorter/sorted-by? tmp-shuffled-bam-file) => anything
+        (with-reader sorter/sorted-by? tmp-coordinate-sorted-sam-file) => truthy
+        (with-reader sorter/sorted-by? tmp-coordinate-sorted-bam-file) => truthy
+        (with-reader sorter/sorted-by? tmp-queryname-sorted-sam-file) =future=> truthy
+        (with-reader sorter/sorted-by? tmp-queryname-sorted-bam-file) =future=> truthy
+        (with-reader sorter/sort-order tmp-shuffled-sam-file) => #(#{:queryname :coordinate :unsorted :unknown} %)
+        (with-reader sorter/sort-order tmp-shuffled-bam-file) => #(#{:queryname :coordinate :unsorted :unknown} %)
+        (with-reader sorter/sort-order tmp-coordinate-sorted-sam-file) => sorter/order-coordinate
+        (with-reader sorter/sort-order tmp-coordinate-sorted-bam-file) => sorter/order-coordinate
+        (with-reader sorter/sort-order tmp-queryname-sorted-sam-file) =future=> sorter/order-queryname
+        (with-reader sorter/sort-order tmp-queryname-sorted-bam-file) =future=> sorter/order-queryname
         ))


remove io clj

remove io clj

--- clojure/src/main/resources/clojure/java/io.clj
+++ clojure/src/main/resources/clojure/java/io.clj
@@ -1,441 +0,0 @@
-;   Copyright (c) Rich Hickey. All rights reserved.
-;   The use and distribution terms for this software are covered by the
-;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
-;   which can be found in the file epl-v10.html at the root of this distribution.
-;   By using this software in any fashion, you are agreeing to be bound by
-;   the terms of this license.
-;   You must not remove this notice, or any other, from this software.
-
-(ns
-  ^{:author "Stuart Sierra, Chas Emerick, Stuart Halloway",
-    :doc "This file defines polymorphic I/O utility functions for Clojure."}
-  clojure.java.io
-  (:require clojure.string)
-  (:import
-    (java.io Reader InputStream InputStreamReader PushbackReader
-                    BufferedReader File OutputStream
-                    OutputStreamWriter BufferedWriter Writer
-                    FileInputStream FileOutputStream ByteArrayOutputStream
-                    StringReader ByteArrayInputStream
-                    BufferedInputStream BufferedOutputStream
-                    CharArrayReader Closeable)
-    (java.net URI URL MalformedURLException Socket URLDecoder URLEncoder)))
-
-#_(def
-  ^{:doc "Type object for a Java primitive byte array."
-    :private true
-    }
-  byte-array-type (class (make-array Byte/TYPE 0)))
-
-#_(def
-  ^{:doc "Type object for a Java primitive char array."
-    :private true}
-  char-array-type (class (make-array Character/TYPE 0)))
-
-#_(defprotocol ^{:added "1.2"} Coercions
-  "Coerce between various 'resource-namish' things."
-  (^{:tag java.io.File, :added "1.2"} as-file [x] "Coerce argument to a file.")
-  (^{:tag java.net.URL, :added "1.2"} as-url [x] "Coerce argument to a URL."))
-
-#_(defn- escaped-utf8-urlstring->str [s]
-  (-> (clojure.string/replace s "+" (URLEncoder/encode "+" "UTF-8"))
-    (URLDecoder/decode "UTF-8")))
-
-#_(extend-protocol Coercions
-  nil
-  (as-file [_] nil)
-  (as-url [_] nil)
-
-  String
-  (as-file [s] (File. s))
-  (as-url [s] (URL. s))
-
-  File
-  (as-file [f] f)
-  (as-url [f] (.toURL (.toURI f)))
-
-  URL
-  (as-url [u] u)
-  (as-file [u]
-    (if (= "file" (.getProtocol u))
-      (as-file (escaped-utf8-urlstring->str
-                 (.replace (.getFile u) \/ File/separatorChar)))
-      (throw (IllegalArgumentException. (str "Not a file: " u)))))
-
-  URI
-  (as-url [u] (.toURL u))
-  (as-file [u] (as-file (as-url u))))
-
-#_(defprotocol ^{:added "1.2"} IOFactory
-  "Factory functions that create ready-to-use, buffered versions of
-   the various Java I/O stream types, on top of anything that can
-   be unequivocally converted to the requested kind of stream.
-
-   Common options include
-
-     :append    true to open stream in append mode
-     :encoding  string name of encoding to use, e.g. \"UTF-8\".
-
-   Callers should generally prefer the higher level API provided by
-   reader, writer, input-stream, and output-stream."
-  (^{:added "1.2"} make-reader [x opts] "Creates a BufferedReader. See also IOFactory docs.")
-  (^{:added "1.2"} make-writer [x opts] "Creates a BufferedWriter. See also IOFactory docs.")
-  (^{:added "1.2"} make-input-stream [x opts] "Creates a BufferedInputStream. See also IOFactory docs.")
-  (^{:added "1.2"} make-output-stream [x opts] "Creates a BufferedOutputStream. See also IOFactory docs."))
-
-#_(defn ^Reader reader
-  "Attempts to coerce its argument into an open java.io.Reader.
-   Default implementations always return a java.io.BufferedReader.
-
-   Default implementations are provided for Reader, BufferedReader,
-   InputStream, File, URI, URL, Socket, byte arrays, character arrays,
-   and String.
-
-   If argument is a String, it tries to resolve it first as a URI, then
-   as a local file name.  URIs with a 'file' protocol are converted to
-   local file names.
-
-   Should be used inside with-open to ensure the Reader is properly
-   closed."
-  {:added "1.2"}
-  [x & opts]
-  (make-reader x (when opts (apply hash-map opts))))
-
-#_(defn ^Writer writer
-  "Attempts to coerce its argument into an open java.io.Writer.
-   Default implementations always return a java.io.BufferedWriter.
-
-   Default implementations are provided for Writer, BufferedWriter,
-   OutputStream, File, URI, URL, Socket, and String.
-
-   If the argument is a String, it tries to resolve it first as a URI, then
-   as a local file name.  URIs with a 'file' protocol are converted to
-   local file names.
-
-   Should be used inside with-open to ensure the Writer is properly
-   closed."
-  {:added "1.2"}
-  [x & opts]
-  (make-writer x (when opts (apply hash-map opts))))
-
-#_(defn ^InputStream input-stream
-  "Attempts to coerce its argument into an open java.io.InputStream.
-   Default implementations always return a java.io.BufferedInputStream.
-
-   Default implementations are defined for OutputStream, File, URI, URL,
-   Socket, byte array, and String arguments.
-
-   If the argument is a String, it tries to resolve it first as a URI, then
-   as a local file name.  URIs with a 'file' protocol are converted to
-   local file names.
-
-   Should be used inside with-open to ensure the InputStream is properly
-   closed."
-  {:added "1.2"}
-  [x & opts]
-  (make-input-stream x (when opts (apply hash-map opts))))
-
-#_(defn ^OutputStream output-stream
-  "Attempts to coerce its argument into an open java.io.OutputStream.
-   Default implementations always return a java.io.BufferedOutputStream.
-
-   Default implementations are defined for OutputStream, File, URI, URL,
-   Socket, and String arguments.
-
-   If the argument is a String, it tries to resolve it first as a URI, then
-   as a local file name.  URIs with a 'file' protocol are converted to
-   local file names.
-
-   Should be used inside with-open to ensure the OutputStream is
-   properly closed."
-  {:added "1.2"}
-  [x & opts]
-  (make-output-stream x (when opts (apply hash-map opts))))
-
-#_(defn- ^Boolean append? [opts]
-  (boolean (:append opts)))
-
-#_(defn- ^String encoding [opts]
-  (or (:encoding opts) "UTF-8"))
-
-#_(defn- buffer-size [opts]
-  (or (:buffer-size opts) 1024))
-
-#_(def default-streams-impl
-  {:make-reader (fn [x opts] (make-reader (make-input-stream x opts) opts))
-   :make-writer (fn [x opts] (make-writer (make-output-stream x opts) opts))
-   :make-input-stream (fn [x opts]
-                        (throw (IllegalArgumentException.
-                                 (str "Cannot open <" (pr-str x) "> as an InputStream."))))
-   :make-output-stream (fn [x opts]
-                         (throw (IllegalArgumentException.
-                                  (str "Cannot open <" (pr-str x) "> as an OutputStream."))))})
-
-#_(defn- inputstream->reader
-  [^InputStream is opts]
-  (make-reader (InputStreamReader. is (encoding opts)) opts))
-
-#_(defn- outputstream->writer
-  [^OutputStream os opts]
-  (make-writer (OutputStreamWriter. os (encoding opts)) opts))
-
-#_(extend BufferedInputStream
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [x opts] x)
-    :make-reader inputstream->reader))
-
-#_(extend InputStream
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [x opts] (BufferedInputStream. x))
-    :make-reader inputstream->reader))
-
-#_(extend Reader
-  IOFactory
-  (assoc default-streams-impl
-    :make-reader (fn [x opts] (BufferedReader. x))))
-
-#_(extend BufferedReader
-  IOFactory
-  (assoc default-streams-impl
-    :make-reader (fn [x opts] x)))
-
-#_(extend Writer
-  IOFactory
-  (assoc default-streams-impl
-    :make-writer (fn [x opts] (BufferedWriter. x))))
-
-#_(extend BufferedWriter
-  IOFactory
-  (assoc default-streams-impl
-    :make-writer (fn [x opts] x)))
-
-#_(extend OutputStream
-  IOFactory
-  (assoc default-streams-impl
-    :make-output-stream (fn [x opts] (BufferedOutputStream. x))
-    :make-writer outputstream->writer))
-
-#_(extend BufferedOutputStream
-  IOFactory
-  (assoc default-streams-impl
-    :make-output-stream (fn [x opts] x)
-    :make-writer outputstream->writer))
-
-#_(extend File
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [^File x opts] (make-input-stream (FileInputStream. x) opts))
-    :make-output-stream (fn [^File x opts] (make-output-stream (FileOutputStream. x (append? opts)) opts))))
-
-#_(extend URL
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [^URL x opts]
-                         (make-input-stream
-                           (if (= "file" (.getProtocol x))
-                             (FileInputStream. (as-file x))
-                             (.openStream x)) opts))
-    :make-output-stream (fn [^URL x opts]
-                          (if (= "file" (.getProtocol x))
-                            (make-output-stream (as-file x) opts)
-                            (throw (IllegalArgumentException. (str "Can not write to non-file URL <" x ">")))))))
-
-#_(extend URI
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [^URI x opts] (make-input-stream (.toURL x) opts))
-    :make-output-stream (fn [^URI x opts] (make-output-stream (.toURL x) opts))))
-
-#_(extend String
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [^String x opts]
-                         (try
-                           (make-input-stream (URL. x) opts)
-                           (catch MalformedURLException e
-                             (make-input-stream (File. x) opts))))
-    :make-output-stream (fn [^String x opts]
-                          (try
-                            (make-output-stream (URL. x) opts)
-                            (catch MalformedURLException err
-                              (make-output-stream (File. x) opts))))))
-
-#_(extend Socket
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [^Socket x opts] (make-input-stream (.getInputStream x) opts))
-    :make-output-stream (fn [^Socket x opts] (make-output-stream (.getOutputStream x) opts))))
-
-#_(extend byte-array-type
-  IOFactory
-  (assoc default-streams-impl
-    :make-input-stream (fn [x opts] (make-input-stream (ByteArrayInputStream. x) opts))))
-
-#_(extend char-array-type
-  IOFactory
-  (assoc default-streams-impl
-    :make-reader (fn [x opts] (make-reader (CharArrayReader. x) opts))))
-
-#_(extend Object
-  IOFactory
-  default-streams-impl)
-
-#_(defmulti
-  ^{:doc "Internal helper for copy"
-    :private true
-    :arglists '([input output opts])}
-  do-copy
-  (fn [input output opts] [(type input) (type output)]))
-
-#_(defmethod do-copy [InputStream OutputStream] [^InputStream input ^OutputStream output opts]
-  (let [buffer (make-array Byte/TYPE (buffer-size opts))]
-    (loop []
-      (let [size (.read input buffer)]
-        (when (pos? size)
-          (do (.write output buffer 0 size)
-            (recur)))))))
-
-#_(defmethod do-copy [InputStream Writer] [^InputStream input ^Writer output opts]
-  (let [^"[C" buffer (make-array Character/TYPE (buffer-size opts))
-        in (InputStreamReader. input (encoding opts))]
-    (loop []
-      (let [size (.read in buffer 0 (alength buffer))]
-        (if (pos? size)
-          (do (.write output buffer 0 size)
-            (recur)))))))
-
-#_(defmethod do-copy [InputStream File] [^InputStream input ^File output opts]
-  (with-open [out (FileOutputStream. output)]
-    (do-copy input out opts)))
-
-#_(defmethod do-copy [Reader OutputStream] [^Reader input ^OutputStream output opts]
-  (let [^"[C" buffer (make-array Character/TYPE (buffer-size opts))
-        out (OutputStreamWriter. output (encoding opts))]
-    (loop []
-      (let [size (.read input buffer)]
-        (if (pos? size)
-          (do
-            (.write out buffer 0 size)
-            (recur))
-          (.flush out))))))
-
-#_(defmethod do-copy [Reader Writer] [^Reader input ^Writer output opts]
-  (let [^"[C" buffer (make-array Character/TYPE (buffer-size opts))]
-    (loop []
-      (let [size (.read input buffer)]
-        (when (pos? size)
-          (do (.write output buffer 0 size)
-            (recur)))))))
-
-#_(defmethod do-copy [Reader File] [^Reader input ^File output opts]
-  (with-open [out (FileOutputStream. output)]
-    (do-copy input out opts)))
-
-#_(defmethod do-copy [File OutputStream] [^File input ^OutputStream output opts]
-  (with-open [in (FileInputStream. input)]
-    (do-copy in output opts)))
-
-#_(defmethod do-copy [File Writer] [^File input ^Writer output opts]
-  (with-open [in (FileInputStream. input)]
-    (do-copy in output opts)))
-
-#_(defmethod do-copy [File File] [^File input ^File output opts]
-  (with-open [in (-> input FileInputStream. .getChannel)
-              out (-> output FileOutputStream. .getChannel)]
-    (let [sz (.size in)]
-      (loop [pos 0]
-        (let [bytes-xferred (.transferTo in pos (- sz pos) out)
-              pos (+ pos bytes-xferred)]
-          (when (< pos sz)
-            (recur pos)))))))
-
-#_(defmethod do-copy [String OutputStream] [^String input ^OutputStream output opts]
-  (do-copy (StringReader. input) output opts))
-
-#_(defmethod do-copy [String Writer] [^String input ^Writer output opts]
-  (do-copy (StringReader. input) output opts))
-
-#_(defmethod do-copy [String File] [^String input ^File output opts]
-  (do-copy (StringReader. input) output opts))
-
-#_(defmethod do-copy [char-array-type OutputStream] [input ^OutputStream output opts]
-  (do-copy (CharArrayReader. input) output opts))
-
-#_(defmethod do-copy [char-array-type Writer] [input ^Writer output opts]
-  (do-copy (CharArrayReader. input) output opts))
-
-#_(defmethod do-copy [char-array-type File] [input ^File output opts]
-  (do-copy (CharArrayReader. input) output opts))
-
-#_(defmethod do-copy [byte-array-type OutputStream] [^"[B" input ^OutputStream output opts]
-  (do-copy (ByteArrayInputStream. input) output opts))
-
-#_(defmethod do-copy [byte-array-type Writer] [^"[B" input ^Writer output opts]
-  (do-copy (ByteArrayInputStream. input) output opts))
-
-#_(defmethod do-copy [byte-array-type File] [^"[B" input ^Writer output opts]
-  (do-copy (ByteArrayInputStream. input) output opts))
-
-#_(defn copy
-  "Copies input to output.  Returns nil or throws IOException.
-  Input may be an InputStream, Reader, File, byte[], or String.
-  Output may be an OutputStream, Writer, or File.
-
-  Options are key/value pairs and may be one of
-
-    :buffer-size  buffer size to use, default is 1024.
-    :encoding     encoding to use if converting between
-                  byte and char streams.
-
-  Does not close any streams except those it opens itself
-  (on a File)."
-  {:added "1.2"}
-  [input output & opts]
-  (do-copy input output (when opts (apply hash-map opts))))
-
-#_(defn ^String as-relative-path
-  "Take an as-file-able thing and return a string if it is
-   a relative path, else IllegalArgumentException."
-  {:added "1.2"}
-  [x]
-  (let [^File f (as-file x)]
-    (if (.isAbsolute f)
-      (throw (IllegalArgumentException. (str f " is not a relative path")))
-      (.getPath f))))
-
-#_(defn ^File file
-  "Returns a java.io.File, passing each arg to as-file.  Multiple-arg
-   versions treat the first argument as parent and subsequent args as
-   children relative to the parent."
-  {:added "1.2"}
-  ([arg]
-    (as-file arg))
-  ([parent child]
-    (File. ^File (as-file parent) ^String (as-relative-path child)))
-  ([parent child & more]
-    (reduce file (file parent child) more)))
-
-#_(defn delete-file
-  "Delete file f. Raise an exception if it fails unless silently is true."
-  {:added "1.2"}
-  [f & [silently]]
-  (or (.delete (file f))
-    silently
-    (throw (java.io.IOException. (str "Couldn't delete " f)))))
-
-#_(defn make-parents
-  "Given the same arg(s) as for file, creates all parent directories of
-   the file they represent."
-  {:added "1.2"}
-  [f & more]
-  (when-let [parent (.getParentFile ^File (apply file f more))]
-    (.mkdirs parent)))
-
-#_(defn ^URL resource
-  "Returns the URL for a named resource. Use the context class loader
-   if no loader is specified."
-  {:added "1.2"}
-  ([n] (resource n (.getContextClassLoader (Thread/currentThread))))
-  ([n ^ClassLoader loader] (.getResource loader n)))


Added queries to support country counts for spark graphs on GFW site

--- src/clj/forma/hadoop/jobs/cdm.clj
+++ src/clj/forma/hadoop/jobs/cdm.clj
@@ -98,3 +98,35 @@
         (latlon-valid? ?lat ?lon)
         (latlon->tile ?lat ?lon zoom :> ?x ?y ?z))))

+(defn spark-hits
+  "Prep for generate counts by country, for spark graphs on GFW site.
+
+   Example usage:
+    (spark-hits (hfs-seqfile \"/home/dan/Dropbox/local/output\")
+                (hfs-seqfile \"/tmp/forma/data/gadm-path\")
+                -9999.0
+                17
+                \"16\"
+                \"32\"
+                \"2005-12-31\"
+                50)"
+  [src gadm-src nodata tres tres-out start thresh]
+  (let [epoch (date/datetime->period tres-out "2000-01-01")
+        start-period (date/datetime->period tres start)]
+    (<- [?iso ?s-res ?modh ?modv ?s ?l ?p]
+        (src ?sres ?modh ?modv ?s ?l ?prob-series)
+        (gadm-src _ ?modh ?modv ?s ?l ?gadm)
+        (gadm->iso ?gadm :> ?iso)
+        (o/clean-probs ?prob-series nodata :> ?clean-series)
+        (first-hit thresh ?clean-series :> ?first-hit-idx)
+        (+ start-period ?first-hit-idx :> ?period)
+        (date/convert-period-res tres tres-out ?period :> ?period-new-res)
+        (- ?period-new-res epoch :> ?rp)
+        (min-period ?rp :> ?p))))
+
+(defn spark-graphify
+  [src gadm-src nodata tres tres-out start thresh]
+  (let [spark-src (spark-hits src gadm-src nodata tres tres-out start thresh)]
+    (<- [?iso ?p ?ct]
+        (spark-src ?iso _ _ _ _ _ ?p)
+        (c/distinct-count ?p :> ?ct))))