Build queue to keep track of in-progress builds, allow multiple builds to run in parallel.

--- backend/src/circle/backend/build/run.clj
+++ backend/src/circle/backend/build/run.clj
@@ -10,6 +10,18 @@
   (:use [circle.logging :only (add-file-appender)])
   (:use [circle.util.except :only (throw-if throw-if-not)])
   (:use [clojure.tools.logging :only (with-logs error infof errorf)]))
+
+;; Clojure has queues, they're not very well documented. They work
+;; like other standard clojure structures, and support peek, pop,
+;; conj.
+
+(def queue (ref clojure.lang.PersistentQueue/EMPTY))
+
+(def in-progress (ref #{}))
+(def max-in-progress 5)
+(def run-agent (agent nil))
+
+(declare run-first)

 (defn start* [build]
   (dosync
@@ -32,8 +44,16 @@

 (defn log-result [b]
   (if (build/successful? b)
-    (infof "Build %s successful" (build/build-name b))
+    (do
+      (println "Build successful" (build/build-name b))
+      (infof "Build %s successful" (build/build-name b)))
     (errorf "Build %s failed" (build/build-name b))))
+
+(defn finished [b]
+  (dosync
+   (infof "removing build %s from in-progress" (build/build-name b))
+   (alter in-progress disj b))
+  (run-first))

 (defn run-build [b & {:keys [cleanup-on-failure]
                           :or {cleanup-on-failure true}}]
@@ -59,3 +79,20 @@
      (when (and (-> @b :failed?) cleanup-on-failure)
        (cleanup-nodes b)))))

+(defn may-start-build? []
+  (< (count @in-progress) max-in-progress))
+
+(defn run-first []
+  (dosync
+   (when (may-start-build?)
+     (when-let [b (peek @queue)]
+       (alter queue pop)
+       (alter in-progress conj b)
+       (send-off run-agent run-build b)
+       (infof "dequeing build %s" (build/build-name b))))))
+
+(defn add-build [b]
+  (dosync
+   (alter queue conj b)
+   (infof "adding build %s to the queue. There are %s queued builds" (build/build-name b) (count @queue)))
+  (run-first))


core: take-last

core: take-last

--- data/core.clj
+++ data/core.clj
@@ -1428,3 +1428,13 @@
   {:added "1.0"}
   ([s] (drop-last 1 s))
   ([n s] (map (fn [x _] x) s (drop n s))))
+
+(defn take-last
+  "Returns a seq of the last n items in coll.  Depending on the type
+  of coll may be no better than linear time.  For vectors, see also subvec."
+  {:added "1.0"}
+  [n coll]
+  (loop [s (seq coll), lead (seq (drop n coll))]
+    (if lead
+      (recur (next s) (next lead))
+      s)))


Fix #6 Invalid token: will-display-header-view-for-section:


Make it a vec again

Make it a vec again

--- markov-clj/src/markov_clj/markov.clj
+++ markov-clj/src/markov_clj/markov.clj
@@ -1,10 +1,10 @@
 (ns markov-clj.markov)

 (defn remove-handles [corpus]
-  (map (fn [word] (if (.startsWith word "@")
-                    (subs word 1)
-                    word))
-       corpus))
+  (vec (doall (map (fn [word] (if (.startsWith word "@")
+                            (subs word 1)
+                            word))
+               corpus))))

 (defn parse-corpus [s]
   (remove-handles (clojure.string/split s #"\s+")))


Test for function slurp-

Test for function slurp-

--- test/clj_fileutils/fileutils_test.clj
+++ test/clj_fileutils/fileutils_test.clj
@@ -173,3 +173,11 @@
         (is (= (with-out-str (print-slurp-exception "test.txt")) "Warning: cannot read content of the following file: test.txt\n"))
         (is (= (with-out-str (print-slurp-exception "testdir" "test.txt")) "Warning: cannot read content of the following file: testdir/test.txt\n"))))

+(deftest test-slurp-
+    "Check the function clj-fileutils.fileutils/slurp-."
+    (testing "Check the function clj-fileutils.fileutils/slurp-."
+        (are [x y] (= x (slurp- y))
+            ""           "test/test1.txt"
+            "test\n"     "test/test2.txt"
+            "1\n2\n3\n"  "test/test3.txt")))
+


Uploading final Java lessons, IO methods from Java interop

--- Working-With-The-JVM/jvm.clj
+++ Working-With-The-JVM/jvm.clj
@@ -170,3 +170,70 @@

 ; -----------------------------------------------------------------------------
 ; Commonly Used Java Classes
+
+;; ----------------
+;; The System Class
+
+;; The `System` class is useful for interacting with the environment the program
+;; is running in. It can be used for env variables, I/O, and error output.
+
+;; The most useful functions are:
+;; `exit`, `getenv` and `getProperty`
+
+;; --------------
+;; The Date Class
+
+;; Java has great tools for working with dates - one of which is the
+;; `java.util.Date` class, whose online API docs are here:
+;; http://docs.oracle.com/javase/7/docs/api/java/util/Date.html
+
+;; The main takeaways from the `date` class are:
+;; (1) you can represent dates as literals using a form like:
+#inst "2016-09-19T20:40:02.733-00:00"
+;; (2) you have to use the `java.util.DateFormat` class if you want to
+;; customize how you convert dates to strings or convert strings to dates
+;; (3) if you need to perform tasks like comparing dates or add minutes,
+;; hours, or other units of time to a date, use the clj-time library:
+;; https://github.com/clj-time/clj-time
+
+; -----------------------------------------------------------------------------
+; Files and I/O
+
+;; Java I/O isn't exactly easy or straightforward - Clojure makes it simpler.
+;; https://clojure.github.io/clojure/clojure.java.io-api.html
+
+;; Java splits its IO classes, `java.io.BufferedReader`, `java.io.FileReader`,
+;; `java.io.BufferedWriter`, `java.io.FileWriter`, etc each with their own
+;; implementations of methods like `read`, `close`, `append`, `write`, `flush`,
+;; etc. - there's enough here to get yourself into trouble if not careful.
+
+;; Meanwhile, in Clojure, we reduce this complexity to:
+;; `spit` which writes to a resource, and
+;; `slurp` which reads from one
+
+(spit "/tmp/hercules-todo-list"
+"- kill dat lion brov
+- chop up what nasty nasty multi-headed snake thing")
+
+(slurp "/tmp/hercules-todo-list")
+; => "- kill dat lion brov
+; =>  - chop up what nasty multi-headed snake thing"
+
+;; The `with-open` macro is also helpful - as it automatically closes a resource
+;; at the end of its body, ensuring that we don't tie up resources by forgetting
+;; to manually close the resource.
+
+;; The `reader` function is also useful as an alternative to `slurp` when
+;; we don't want to try to read the resource in full and don't want to
+;; sort out which Java class we need to use.
+
+;; We can combine these with the `line-seq` function to read one line at a time:
+
+(with-open [todo-list-rdr (clojure.java.io/reader "/tmp/hercules-todo-list")]
+  (println (first (line-seq todo-list-rdr))))
+; => - kill dat lion brov
+
+;; For more on IO in Clojure, it can be helpful to refer back to:
+;; https://clojure.github.io/clojure/clojure.java.io-api.html
+;; https://docs.oracle.com/javase/7/docs/api/java/nio/file/package-summary.html
+;; and, http://docs.oracle.com/javase/7/docs/api/java/io/package-summary.html


mock created case correctly, add closed for another issue

--- test/gh_waiting_room/service_test.clj
+++ test/gh_waiting_room/service_test.clj
@@ -2,6 +2,7 @@
   (:require [clojure.test :refer :all]
             [io.pedestal.service.test :refer :all]
             [io.pedestal.service.http :as bootstrap]
+            [gh-waiting-room.github :as github]
             [gh-waiting-room.service :as service]))

 (def service
@@ -9,7 +10,7 @@

 (defn- body-of-home-page
   []
-  (with-redefs [gh-waiting-room.github/viewable-issues
+  (with-redefs [github/viewable-issues
                 (constantly [{:position "1"
                               :id "cldwalker/gh-waiting-room#1"
                               :url "https://github.com/cldwalker/gh-waiting-room/issues/1"}])
@@ -36,7 +37,8 @@
       "Issues can be referenced by their unique id and users know of it."))

 (defn inc-mock-count [mocks-called key]
-  (swap! mocks-called update-in [key] (fnil inc 0)))
+  (fn [& args]
+   (swap! mocks-called update-in [key] (fnil inc 0))))

 (defn fns-called-for-webhook-page
   [action]
@@ -44,12 +46,17 @@
         json {"action" action
               "repository" {"full_name" "cldwalker/some-name"}
               "issue" {"number" "10"}}]
-    (with-redefs [service/update-gh-issues (constantly (inc-mock-count mocks-called :update-gh-issues))
-                  gh-waiting-room.github/create-issue-comment (constantly (inc-mock-count mocks-called :create-issue-comment))]
+    (with-redefs [service/update-gh-issues (inc-mock-count mocks-called :update-gh-issues)
+                  github/create-issue-comment (inc-mock-count mocks-called :create-issue-comment)
+                  github/viewable-issues (constantly [])]
       (service/webhook-page {:json-params json}))
     @mocks-called))

 ;;; Doesn't use response-for as it doesn't support :post yet
 (deftest webhook-page-test
   (is (= (fns-called-for-webhook-page "created")
-         {:update-gh-issues 1 :create-issue-comment 1})))+         {:update-gh-issues 1 :create-issue-comment 1})
+      "Updates issues and creates comment for a newly created issue")
+  (is (= (fns-called-for-webhook-page "closed")
+         {})
+      "Doesn't update issues for an inactive issue that is closed"))


Work around an oddity of (only) Clojure 1.3.

--- test/midje/internal_ideas/t_fakes.clj
+++ test/midje/internal_ideas/t_fakes.clj
@@ -258,11 +258,19 @@
     (handle-mocked-call #'f [1] fakes)    (counts) => [2 0 0]
     (handle-mocked-call #'f [2] fakes)    (counts) => [2 0 1]
     (handle-mocked-call #'g [1] fakes)    (counts) => [2 1 1]))
+)
+;; Closing the binding just above because Clojure 1.3 (and only
+;; Clojure 1.3) becomes confused about unbound vars that are defined
+;; inside of a `binding` scope. The binding in the fact below
+;; causes `bound?` to return true, but dereferencing the var still returns
+;; the magic value #<Unbound Unbound>.

 (def unbound-var)
 (def bound-var 3)
 (def #^:dynamic rebound)
-
+
+(binding [midje.config/*allow-default-prerequisites* true]
+
 (fact "fakes contain the value of their function-var at moment of binding"
   (:value-at-time-of-faking (fake (unbound-var) => 2)) => nil
   (:value-at-time-of-faking (fake (bound-var) => 888)) => 3


Fix a quick regression in the tests

Fix a quick regression in the tests

--- test/picard/test/middleware/retry.clj
+++ test/picard/test/middleware/retry.clj
@@ -73,8 +73,9 @@
                         (downstream :response [status {"content-length" "0"} ""]))
                     (downstream :response [202 {"content-length" "0"} ""])))))
             (middleware/retry {:retries [5 5]})))
+
       (GET "/")
-      (is (= status (last-response-status))))))
+      (is (= status (first (second (first (exchange-events (last-exchange))))))))))

 (deftest doesnt-retry-when-the-body-is-sent
   (with-app


CMR-1600: Additional test for get-keys-in function


fix the add-projects page

fix the add-projects page

--- frontend/controllers/navigation.cljs
+++ frontend/controllers/navigation.cljs
@@ -132,8 +132,8 @@

 (defmethod navigated-to :add-projects
-  [history-imp navigation-point _ state]
-  (assoc state :navigation-point navigation-point :navigation-data {} :navigation-settings {}))
+  [history-imp navigation-point args state]
+  (assoc state :navigation-point navigation-point :navigation-data args :navigation-settings {}))

 (defmethod post-navigated-to! :add-projects
   [history-imp navigation-point _ previous-state current-state]


Use my domain

Use my domain


use memoize to avoid creating extra thread pools


update test

update test

--- test/fileape/core_test.clj
+++ test/fileape/core_test.clj
@@ -20,7 +20,7 @@

 (facts "Test file writing"
        (fact "Test future file name rollover"
-             (create-future-file-name "blabla_abc.gz_000" 0) => "/blabla_abc.gz")
+             (create-future-file-name "blabla_abc.gz_000" 0) => "/blabla_abc.000.gz")

        (fact "test write file"


Test IP validation-only parsing.

Test IP validation-only parsing.

--- test/inet/data/ip_test.clj
+++ test/inet/data/ip_test.clj
@@ -3,13 +3,25 @@
   (:use [clojure.test])
   (:import [java.net InetAddress]))

-(deftest test-inet-address
-  (testing "Round-tripping IPv4 address strings"
-    (is (= "172.12.16.1" (str (ip/address "172.12.16.1")))))
-  (testing "Round-tripping IPv6 address strings"
-    (is (= "fe:1100::1" (str (ip/address "fe:1100:0::1"))))))
+(deftest test-address-validation
+  (testing "Validation"
+    (testing "of IPv4 addresses which are"
+      (is (= true (ip/address? "192.168.1.1")) "valid")
+      (is (= false (ip/address? "8.x.17.y")) "invalid, non-numeric")
+      (is (= false (ip/address? "8.8.256.7")) "invalid, numeric"))
+    (testing "of IPv6 addresses which are"
+      (is (= true (ip/address? "17:fe77::1899:12")) "valid")
+      (is (= false (ip/address? "17::qq")) "invalid, non-numeric")
+      (is (= false (ip/address? "17::18::ae")) "invalid, numeric"))))

-(deftest test-inet-network
+(deftest test-address-roundtrip
+  (testing "Round-tripping"
+    (let [addr "172.12.16.1"]
+      (is (= addr (-> addr ip/address str)) "IPv4 address strings"))
+    (let [addr "fe:1100::1"]
+      (is (= addr (-> addr ip/address str)) "IPv6 address strings"))))
+
+(deftest test-network
   (testing "Creating networks"
     (testing "from IPv4 addresses"
       (let [addr-str   "192.168.0.0"
@@ -19,9 +31,9 @@
                        [addr-obj "InetAddress"]
                        [addr-bytes "bytes"]]]
         (doseq [[addr src] test-data]
-          (is (= "192.168.0.0/32" (str (ip/network addr)))
+          (is (= "192.168.0.0/32" (-> addr ip/network str))
               (format "From %s with implied prefix-length." src))
-          (is (= "192.168.0.0/16" (str (ip/network addr 16)))
+          (is (= "192.168.0.0/16" (-> addr (ip/network 16) str))
               (format "From %s with explicit prefix-length." src)))))
     (testing "from IPv6 addresses"
       (let [addr-str   "fe:11::"
@@ -31,9 +43,9 @@
                         [addr-obj "InetAddress"]
                         [addr-bytes "bytes"]]]
         (doseq [[addr src] test-data]
-          (is (= "fe:11::/128" (str (ip/network addr)))
+          (is (= "fe:11::/128" (-> addr ip/network str))
               (format "From %s with implied prefix-length." src))
-          (is (= "fe:11::/32" (str (ip/network addr 32)))
+          (is (= "fe:11::/32" (-> addr (ip/network 32) str))
               (format "From %s with explicit prefix-length." src)))))))

 (deftest test-network-contains


chapter sample updates

chapter sample updates

--- pcl/chap_16/chap_16.clj
+++ pcl/chap_16/chap_16.clj
@@ -3,7 +3,7 @@

 ; multimethods
 (defmulti draw :shape)
-(defmethod draw :square [shape] "TBD: draw a sqaure")
+(defmethod draw :square [shape] "TBD: draw a square")
 (defmethod draw :circle [shape] "TBD: draw a circle")

 ; using multimethods
@@ -19,18 +19,30 @@

 ; note the implicit deposit: problem?
 (defmethod withdraw :checking [account amount]
-  (let [overdraft
-	(- amount (:balance account))
-	safe-account
-	(if (> overdraft 0)
+  (let [over-account (account :overdraft-account)
+	over-amount (- amount (:balance account))
+	withdrawal-account
+	(if (> over-amount 0)
 	  (merge account
-		 {:overdraft-account (withdraw (account :overdraft-account) overdraft)
+		 {:overdraft-account (withdraw over-account over-amount)
 		  :balance amount})
 	  account)]
-    (raw-withdraw safe-account amount)))
+    (raw-withdraw withdrawal-account amount)))

 ; dispatching on more than one argument
 (defmulti beat (fn [d s] [(:drum d)(:stick s)]))
 (defmethod beat [:snare-drum :brush] [drum stick] "snare drum and brush")
 (defmethod beat [:snare-drum :soft-mallet] [drum stick] "snare drum and soft mallet")
 (defmethod beat :default [drum stick] "default value, if you want one")
+
+(let [original-state {:account-type :bank :balance 100}
+      updated-state (withdraw original-state 50)]
+  (println original-state updated-state))
+
+(let [overdraft {:account-type :checking, :balance 1000}
+      original-state {:account-type :checking
+		      :balance 100
+		      :overdraft-account overdraft}
+      updated-state (withdraw original-state 500)]
+  (println original-state)
+  (println updated-state))


Clarify unsubscribe doc.

Clarify unsubscribe doc.

--- messaging/src/immutant/messaging.clj
+++ messaging/src/immutant/messaging.clj
@@ -339,8 +339,7 @@

    If no context is provided, a new context is created for this
    action. If a context is provided, it must have its :subscription-name
-   set to the same value used when creating the subscription. See
-   [[subscribe]].
+   set to the same value given to the context passed to [[subscribe]].

    The following options are supported [default]:


cleanup update-in benchmark, now w/in 3X of Clojure JVM on V8

--- benchmark/cljs/benchmark_runner.cljs
+++ benchmark/cljs/benchmark_runner.cljs
@@ -88,8 +88,8 @@
 (println)

 (println ";; update-in")
-(simple-benchmark [coll {:foo 1}] (update-in coll [:x] inc) 1000000)
-(simple-benchmark [coll {:foo 1} ks (array-seq (array :x))] (update-in coll ks inc) 1000000)
+(simple-benchmark [coll {:foo 1} ks [:foo]] (update-in coll ks inc) 1000000)
+(simple-benchmark [coll (array-map :foo 1) ks [:foo]] (update-in coll ks inc) 1000000)
 (println)

 (println ";;; obj-map")


note that `:as-jars` option requires special container

--- boot/core/src/boot/task/built_in.clj
+++ boot/core/src/boot/task/built_in.clj
@@ -436,7 +436,9 @@
   scope options may be used to add or remove scope(s) from this set.

   The as-jars option pulls in dependency jars without exploding them, such that
-  the jarfiles themselves are copied into the fileset.
+  the jarfiles themselves are copied into the fileset. When using as-jars you
+  need a special classloader like a servlet container (e.g. Tomcat, Jetty) that
+  picks up the packaged jars and places them on the classpath.

   When jars are exploded, the include and exclude options control what paths are
   added to the uberjar, with a path only being added if it matches an include


No more gorillarepl commentedoutness

No more gorillarepl commentedoutness

--- link/.lein/profiles.clj
+++ link/.lein/profiles.clj
@@ -17,26 +17,3 @@
                        [pjstadig/humane-test-output "0.7.0"]
                        [slamhound "1.5.5"]
                        [spyscope "0.1.5"]]}}
-
-;;;; WIP: more emacs-like Gorilla keybindings.
-;; :gorilla-options {:keymap { ;; "command:app:commands" ""
-;;                            ;; "command:worksheet:leaveBack" "alt+p"
-;;                            ;; "command:worksheet:leaveForward" "alt+n"
-;;                            "command:evaluator:evaluate" "ctrl+c ctrl+e"
-;;                            "command:evaluator:evaluate-all" "ctrl+c ctrl+k"
-;;                            ;; "command:worksheet:clear-output" ""
-;;                            ;; "command:worksheet:clear-all" ""
-;;                            ;; "command:worksheet:delete" ""
-;;                            ;; "command:worksheet:changeToFree" ""
-;;                            ;; "command:worksheet:changeToCode" ""
-;;                            "command:app:open" "ctrl+x ctrl+f"
-;;                            "command:app:save" "ctrl+x ctrl+s"
-;;                            "command:app:saveas" "ctrl+x ctrl+w"
-;;                            "command:worksheet:newBelow" "ctrl+o"
-;;                            ;; "command:worksheet:newAbove" ""
-;;                            ;; "command:worksheet:moveUp" ""
-;;                            ;; "command:worksheet:moveDown" ""
-;;                            "command:docs:clojuredocs" "ctrl+c ctrl+d"
-;;                            ;; "command:app:reset-worksheet" ""
-;;                            ;; "command:worksheet:completions" ""
-;;                            }}


CMR-3298 removes commented out lines

CMR-3298 removes commented out lines

--- access-control-app/src/cmr/access_control/services/acl_service.clj
+++ access-control-app/src/cmr/access_control/services/acl_service.clj
@@ -107,7 +107,6 @@
   ([context acl action]
    (acl-log-message context acl nil action))
   ([context new-acl existing-acl action]
-   ;(let [user (context->user-id context)])
    (let [user (tokens/get-user-id context (:token context))]
      (cond
        (= action :create) (format "User: [%s] Created ACL [%s]" user new-acl)
@@ -161,15 +160,11 @@
   [_]
   (cpv/merge-params-config
     cpv/basic-params-config
-    {:single-value #{:include-full-acl}
+    {:single-value #{}
      :multiple-value #{:permitted-group :identity-type :provider}
      :always-case-sensitive #{}
      :disallow-pattern #{:identity-type :permitted-user :group-permission}
      :allow-or #{}}))
-
-(defmethod cpv/valid-query-level-params :acl
-  [_]
-  #{:include-full-acl})

 (defmethod cpv/valid-parameter-options :acl
   [_]
@@ -279,17 +274,6 @@
       [(format (str "Parameter identity_type has invalid values [%s]. "
                     "Only 'provider', 'system', 'single_instance', or 'catalog_item' can be specified.")
                (str/join ", " invalid-types))])))
-
-(defn boolean-value-validation
-  "Validates that all of the boolean parameters have values of true or false."
-  [concept-type params]
-  (let [bool-params (select-keys params [:include-full-acl])]
-    (mapcat
-      (fn [[param value]]
-        (when-not (contains? #{"true" "false"} (when value (str/lower-case value)))
-          [(format "Parameter %s must take value of true or false but was [%s]"
-                   (csk/->snake_case_string param) value)]))
-      bool-params)))

 (defn validate-acl-search-params
   "Validates the parameters for an ACL search. Returns the parameters or throws an error if invalid."
@@ -305,10 +289,7 @@
     (cpv/validate-parameters
       :acl safe-params
       (concat cpv/common-validations
-              [permitted-group-validation
-               identity-type-validation
-               group-permission-validation
-               boolean-value-validation])
+              [permitted-group-validation identity-type-validation group-permission-validation])
       type-errors))
   params)

@@ -327,16 +308,6 @@
    :provider :string
    :permitted-user :acl-permitted-user
    :group-permission :acl-group-permission})
-
-(defmethod cp/parse-query-level-params :acl
-  [concept-type params]
-  (let [[params query-attribs] (cp/default-parse-query-level-params :acl params)
-        result-features (when (= (:include-full-acl params) "true"
-                                 [:include-full-acl]))]
-    [(dissoc params :include-full-acl)
-     (merge query-attribs
-            (when (= (:include-full-acl params) "true")
-              {:result-features [:include-full-acl]}))]))

 (defmethod cp/parameter->condition :acl-identity-type
  [context concept-type param value options]
@@ -439,6 +410,10 @@
 (def all-permissions
   "The set of all permissions checked and returned by the functions below."
   #{:read :order :update :delete})
+
+(def provider-level-permissions
+  "The set of permissions that are checked at the provider level."
+  #{:update :delete})

 (defn- collect-permissions
   "Returns seq of any permissions where (grants-permission? acl permission) returns true for any acl in acls."
@@ -492,41 +467,20 @@
   "Returns true if permission keyword is granted on concept to any sids by given acl."
   [acl permission concept sids]
   (and (acl/acl-matches-sids-and-permission? sids (name permission) acl)
-       (case (:concept-type concept)
-         :collection (acl-matchers/coll-applicable-acl? (:provider-id concept) concept acl)
-         :granule (acl-matches-granule? acl concept))))
-
-(defn- provider-acl?
-  "Returns true if the ECHO-style acl specifically identifies the given provider id."
-  [provider-id acl]
-  (or
-    (-> acl :provider-object-identity :provider-id (= provider-id))
-    (-> acl :catalog-item-identity :provider-id (= provider-id))))
-
-(defn- ingest-management-acl?
-  "Returns true if the ACL targets a provider INGEST_MANAGEMENT_ACL."
-  [acl]
-  (-> acl :provider-object-identity :target (= schema/ingest-management-acl-target)))
+       (if (contains? provider-level-permissions permission)
+         (when-let [acl-provider-id (-> acl :provider-object-identity :provider-id)]
+           (= acl-provider-id (:provider-id concept)))
+         ;; else check that the concept matches
+         (case (:concept-type concept)
+           :collection (acl-matchers/coll-applicable-acl? (:provider-id concept) concept acl)
+           :granule (acl-matches-granule? acl concept)))))

 (defn- concept-permissions-granted-by-acls
-  "Returns the set of permission keywords (:read, :order, and :update) granted on concept
+  "Returns the set of permission keywords (:read, :update, :order, or :delete) granted on concept
    to the seq of group guids by seq of acls."
   [concept sids acls]
-  (let [provider-acls (filter #(provider-acl? (:provider-id concept) %) acls)
-        ;; When a user has UPDATE on the provider's INGEST_MANAGEMENT_ACL target, then they have UPDATE and
-        ;; DELETE permission on all of the provider's catalog items.
-        ingest-management-permissions (when (some #(acl/acl-matches-sids-and-permission? sids "update" %)
-                                                  (filter ingest-management-acl? provider-acls))
-                                        [:update :delete])
-        ;; The remaining catalog item ACLs can only grant READ or ORDER permission.
-        catalog-item-acls (filter :catalog-item-identity provider-acls)
-        catalog-item-permissions (for [permission [:read :order]
-                                       :when (some #(grants-concept-permission? % permission concept sids)
-                                                   catalog-item-acls)]
-                                   permission)]
-    (set
-      (concat catalog-item-permissions
-              ingest-management-permissions))))
+  (collect-permissions #(grants-concept-permission? %1 %2 concept sids)
+                       acls))

 (defn- add-acl-enforcement-fields
   "Adds all fields necessary for comparing concept map against ACLs."
@@ -538,7 +492,7 @@
                        (mdb/get-latest-concept context parent-id)))
       concept)))

-(defn get-catalog-item-permissions
+(defn get-concept-permissions
   "Returns a map of concept ids to seqs of permissions granted on that concept for the given username."
   [context username-or-type concept-ids]
   (let [sids (get-sids context username-or-type)
@@ -552,13 +506,10 @@
 (defn- system-permissions-granted-by-acls
   "Returns a set of permission keywords granted on the system target to the given sids by the given acls."
   [system-object-target sids acls]
-  (let [relevant-acls (filter #(-> % :system-object-identity :target (= system-object-target))
-                              acls)]
-    (set
-      (for [permission [:create :read :update :delete]
-            :when (some #(acl/acl-matches-sids-and-permission? sids (name permission) %)
-                        relevant-acls)]
-        permission))))
+  (collect-permissions (fn [acl permission]
+                         (and (= system-object-target (:target (:system-object-identity acl)))
+                              (acl/acl-matches-sids-and-permission? sids (name permission) acl)))
+                       acls))

 (defn get-system-permissions
   "Returns a map of the system object type to the set of permissions granted to the given username or user type."


Fix change in method signature

Fix change in method signature

--- common/clojure/nightweb/torrents.clj
+++ common/clojure/nightweb/torrents.clj
@@ -162,6 +162,7 @@
                    true ;autoStart
                    nil ;refreshDelay
                    nil ;startDelay
+                   nil ;pageSize
                    nil ;seedPct
                    nil ;eepHost
                    nil ;eepPort


Sort generated run rows by status

Sort generated run rows by status

--- src-cljs/frontend/components/pieces/run_row.cljs
+++ src-cljs/frontend/components/pieces/run_row.cljs
@@ -620,16 +620,28 @@
          ""))
      (gen/vector (gen/elements lorem-words))))

+  ;; https://stackoverflow.com/questions/25324082/index-of-vector-in-clojurescript/32885837#32885837
+  (defn- index-of [coll value]
+    (some (fn [[idx item]] (if (= value item) idx))
+          (map-indexed vector coll)))
+
   (defcard generated-run-rows
     (binding [om-next/*shared* {:timer-atom (timer/initialize)}]
-      (html
-       [:div
-        (let [data (gc/morph-data run-row :run/entity {:run/name #(dashed-lorem)
-                                                       :trigger-info/branch #(dashed-lorem)
-                                                       :trigger-info/subject #(lorem-sentence)})]
-          (if data
-            (card/collection (map run-row data))
-            "Infinite morphs!"))])))
+      (let [statuses [:run-status/needs-setup
+                      :run-status/not-run
+                      :run-status/running
+                      :run-status/succeeded
+                      :run-status/failed
+                      :run-status/canceled]
+            sort-by-status (partial sort-by (comp (partial index-of statuses) :run/status))]
+        (html
+         [:div
+          (let [data (gc/morph-data run-row :run/entity {:run/name #(dashed-lorem)
+                                                         :trigger-info/branch #(dashed-lorem)
+                                                         :trigger-info/subject #(lorem-sentence)})]
+            (if data
+              (card/collection (map run-row (sort-by-status data)))
+              "Infinite morphs!"))]))))

   (defcard loading-run-row
     (loading-run-row)))


Whitespace

Whitespace

--- clojure/src/rabbitmq/tutorials/new_task.clj
+++ clojure/src/rabbitmq/tutorials/new_task.clj
@@ -4,7 +4,6 @@
             [langohr.queue :as lq]
             [langohr.basic :as lb]
             [clojure.string :as s]))
-

 (defn -main
   [& args]


cleanup english morphology processing: move stuff from (fo) to (get-string), where most work should take place

--- src/babel/english/morphology.cljc
+++ src/babel/english/morphology.cljc
@@ -19,21 +19,6 @@
    input

    (and (map? input)
-        (get-in input [:english :a :english])
-        (get-in input [:english :b :english])
-        (or (map? (get-in input [:english :a :english]))
-            (map? (get-in input [:english :b :english]))))
-   (string/join " "
-                (list (fo (get-in input [:english :a :english]))
-                      (fo (get-in input [:english :b :english]))))
-   (and (map? input)
-        (get-in input [:english :a])
-        (get-in input [:english :b]))
-   (string/join " "
-                (list (fo (get-in input [:english :a]))
-                      (fo (get-in input [:english :b]))))
-
-   (and (map? input)
         (map? (get-in input [:english])))
    (get-string (get-in input [:english]))

@@ -41,13 +26,6 @@
    (and (map? input)
         (get-in input [:english]))
    (get-string input)
-
-   (and (map? input)
-        (get-in input [:a])
-        (get-in input [:b]))
-   (str (string/join " "
-                     (list (fo (get-in input [:a]))
-                           (fo (get-in input [:b])))))

    (or (seq? input)
        (vector? input))
@@ -160,19 +138,10 @@
                       :b (get-in word '(:b :b))}})
    (and
     (get-in word '(:a))
-    (get-in word '(:b))
-    (string? (get-in word '(:a)))
-    (string? (get-in word '(:b))))
-   (join " "
-         (list (get-in word '(:a))
-               (get-in word '(:b))))
-
-   (and
-    (get-in word '(:a))
     (get-in word '(:b)))
    (join " "
-         (list (fo (get-in word '(:a)))
-               (fo (get-in word '(:b)))))
+         (list (get-string (get-in word '(:a)))
+               (get-string (get-in word '(:b)))))

    ;; TODO: this seems wrong: how could :infl == :english?
    (and (= :english (get-in word '(:infl)))
@@ -180,13 +149,15 @@
    (get-in word '(:english))

    (= true (get-in word '(:hidden)))
-;;   "Ø"
+   ;;   "Ø"
    ""
+
    (and
     (= true (get-in word '(:a :hidden)))
     (= true (get-in word '(:b :hidden))))
-;;   "Ø"
+   ;;   "Ø"
    ""
+
    (= true (get-in word '(:a :hidden)))
    (get-string (get-in word '(:b)))


improved photo import: date/time digitized


For clj, move output specific part to out-format-fn.

--- src/dsbdp/data_processing_dsl.clj
+++ src/dsbdp/data_processing_dsl.clj
@@ -89,14 +89,14 @@
                                                       nested-expr-ret-tmp (last nested-expr-tmp)
                                                       nested-expr-ret [nested-expr-ret-tmp (prefix-rule-name '__offset-increment (inc nesting-level))]
                                                       nested-expr (into '() (reverse (assoc (vec nested-expr-tmp) (- (count nested-expr-tmp) 1) nested-expr-ret)))
-                                                      loop-expr `(loop [~'offset (~seq-params :initial-offset) ~'result-vec []]
+                                                      loop-expr `(loop [~'offset (~seq-params :initial-offset) ~'result ~(out-format-fn)]
                                                                   (let [~'tmp-result ~nested-expr
                                                                         ~'new-offset (+ ~'offset (second ~'tmp-result))
-                                                                        ~'new-result-vec (conj ~'result-vec (first ~'tmp-result))]
+                                                                        ~'new-result ~(out-format-fn 'result '(first tmp-result))]
 ;                                                                    (println "Nested seq step:" ~'new-offset "--" (count ~'input))
                                                                     (if (< ~'new-offset (count ~'input))
-                                                                      (recur ~'new-offset ~'new-result-vec)
-                                                                      ~'new-result-vec)))]
+                                                                      (recur ~'new-offset ~'new-result)
+                                                                      ~'new-result)))]
 												(conj v
 													  (prefix-rule-name rule-name nesting-level)
 													  loop-expr)))
@@ -159,24 +159,28 @@
     rules))

 (defn- clj-out-format-fn
-  [rules output nesting-level]
-  (reduce
-    (fn [v rule]
-      (cond
-        (sequential? (second rule)) (conj v
-                                          `(assoc
-                                            ~(name (first rule))
-                                            ~(prefix-rule-name (first rule) nesting-level)))
-        (cond-rule-expr? (second rule)) (conj v
-                                              `(assoc ~(name (first rule))
-                                                      ~(prefix-rule-name (first rule) nesting-level)))
-        :default (do (println "Clj Map Body: unknown element for rule:" (str rule))
-                     ;(throw (RuntimeException. "Clj Map Body: unknown element for rule"))
-                     )))
-    (if (nil? output)
-      '[-> {}]
-      '[-> output])
-    rules))
+  ([]
+    [])
+  ([result-vec new-result-value]
+    `(conj ~result-vec ~new-result-value))
+  ([rules output nesting-level]
+    (reduce
+      (fn [v rule]
+        (cond
+          (sequential? (second rule)) (conj v
+                                            `(assoc
+                                              ~(name (first rule))
+                                              ~(prefix-rule-name (first rule) nesting-level)))
+          (cond-rule-expr? (second rule)) (conj v
+                                                `(assoc ~(name (first rule))
+                                                        ~(prefix-rule-name (first rule) nesting-level)))
+          :default (do (println "Clj Map Body: unknown element for rule:" (str rule))
+                       ;(throw (RuntimeException. "Clj Map Body: unknown element for rule"))
+                       )))
+      (if (nil? output)
+        '[-> {}]
+        '[-> output])
+      rules)))

 (defn- csv-str-out-format-fn
   [rules output nesting-level]


CMR-2814 Renames function

CMR-2814 Renames function

--- access-control-app/src/cmr/access_control/services/acl_authorization.clj
+++ access-control-app/src/cmr/access_control/services/acl_authorization.clj
@@ -53,7 +53,7 @@
   [action]
   (format "Permission to %s ACL is denied" (name action)))

-(defn- can?
+(defn- permitted?
   "Returns true if any ACLs grant the current context user the given permission
   keyword (:create, :update, etc.) on the given acl."
   [context permission acl]
@@ -88,5 +88,5 @@
   "Throws service error if user doesn't have permission to intiate a given action for a given acl.
    Actions include create and update."
   [context action acl]
-  (when-not (can? context action acl)
+  (when-not (permitted? context action acl)
     (errors/throw-service-error :bad-request (permission-denied-message action))))


efficient non-dup feature creation

efficient non-dup feature creation

--- src/clojure/zensols/model/execute_classifier.clj
+++ src/clojure/zensols/model/execute_classifier.clj
@@ -92,6 +92,43 @@
   (let [model-conf (model-config)]
     (->> ((:feature-metas-fn model-conf))
          (into {}))))
+
+(def ^{:dynamic true :private true} combined-features nil)
+
+(defmacro with-combined-features
+  {:style/indent 0}
+  [& forms]
+  `(binding [combined-features (atom {})]
+     (do ~@forms)))
+
+(defn- combine-feature-vals
+  "Apply feature creation functions only where they don't already exist
+  in [[combined-features]]."
+  [fmap]
+  (let [cfeats @combined-features
+        fmap (->> fmap
+                  (map (fn [[fkey form]]
+                          (let [feats (or (and cfeats (get cfeats fkey))
+                                          ((eval form)))]
+                            {fkey feats})))
+                  (apply merge))]
+    (log/tracef "combined: %s" (pr-str @combined-features))
+    (swap! combined-features merge fmap)
+    (->> fmap vals (apply merge))))
+
+(defmacro combine-features
+  "Combine features from the forms for which features don't already exist in
+  **features**, which is a map with functions as keys and features already
+  created with that function as the value of the respective function."
+  [& forms]
+  `(->> (merge ~@(->> (map (fn [form]
+                             (->> form first
+                                  (ns-resolve (ns-name *ns*))
+                                  var-get
+                                  (#(hash-map % (read-string (str "#" (pr-str form)))))))
+                           forms)
+                      doall))
+        combine-feature-vals))

 (defn- create-instances
   "Create the weka Instances object using sets of features created from
@@ -277,7 +314,7 @@
                                  (throw (ex-info "No create-features-fn defined in model"
                                                  {:model-name (:name model-conf)})))
           _ (log/debugf "create-features-fn: %s" create-features-fn)
-          cfargs (concat data (list context))
+          cfargs (concat data (if context (list :context context)))
           _ (log/debugf "arg count: %d" (count cfargs))
           features (apply create-features-fn cfargs)]
       (classify-features model features))))


Faster options

Faster options

--- ClojureScript/planck/script/build.clj
+++ ClojureScript/planck/script/build.clj
@@ -3,9 +3,10 @@

 (println "Building")
 (api/build (api/inputs "src")
-  {:output-dir    "out"
-   :output-to     "out/main.js"
-   :optimizations :none
-   :static-fns    true})
+  {:output-dir         "out"
+   :output-to          "out/main.js"
+   :optimizations      :none
+   :static-fns         true
+   :optimize-constants false})
 (println "Done building")
 (System/exit 0)


Robustified the tweet dechunker.

Robustified the tweet dechunker.

--- data_collection/profanity-power-index/src/profanity_power_index/core.clj
+++ data_collection/profanity-power-index/src/profanity_power_index/core.clj
@@ -13,17 +13,16 @@
 ;; Partitioning transducer.
 (defn partitioner-xform []
   (comp
-    ;; Partition by whether it's a tweet terminator.
-    (partition-by #(str/includes? % "\r\n"))
-    ;; Merge the terminator with the rest of the tweet.
-    (partition-all 2)
-    ;; Flatten the nested sequence of chunks.
-    (map flatten)
-    ;; And concatenate into one string.
-    (map #(apply str %))
-    ;; And finally filter out empty strings.
-    (filter #(not (str/blank? %)))))
-
+    ;; Split on the carriage return. These occur only at the end of a tweet.
+    (mapcat (fn [s] (str/split s #"\r")))
+    ;; Now partition the incoming vectors by a blank string.
+    ;; This groups the tweet chunks into vectors, and the boundaries into
+    ;; empty strings.
+    (partition-by str/blank?)
+    ;; Mash everything together into a string.
+    (map (fn [s] (apply str s)))
+    ;; Finally, filter out the newline boundaries.
+    (filter (fn [s] (not (str/blank? s))))))

 ;; (def transfer-chan (chan 5))
 (def transfer-chan (chan 5))
@@ -41,8 +40,7 @@
     ;; This is the 'error' function for the callback. It has a response and
     ;; a throwable.
     (fn [err thr] (binding [*out* *err*]
-      (println "ERROR:" err)
-      (println (str thr))))))
+      (println "ERROR:" err)))))

 ;;;; PROCESSING HELPERS ;;;;
 (defn contains-profanity? [text]
@@ -63,13 +61,20 @@
     (catch Exception e
     (do
       (println e)
-      (println text)
       {:text ""}))))

 (defn parse-filter []
   (comp
     (map safe-parse-json)
-    (filter (fn [t] (-> t :text str/lower-case contains-profanity?)))))
+    (filter
+      (fn [t]
+        (-> t
+            ;; Pull the text out, or an empty string.
+            (:text "")
+            ;; Convert to lower case.
+            str/lower-case
+            ;; Check for profanity.
+            contains-profanity?)))))

 (defn turbine-topology-vector [elastic-url]
   (let [es-conn (esr/connect elastic-url)]
@@ -80,8 +85,8 @@
     ;; but we need to deserialize to execute the filters.
     [:spread :input
       [[:deser1 (parse-filter)]
-      [:deser2 (parse-filter)]
-      [:deser3 (parse-filter)]]]
+       [:deser2 (parse-filter)]
+       [:deser3 (parse-filter)]]]
     ;; Unify all of the parsed values asynchronously.
     [:union [:deser1 :deser2 :deser3]
              [:to-out (map identity)]]
@@ -113,10 +118,9 @@
     ;; Read from the dechunker chan in a go loop and drop into turbine.
     (go-loop []
       (let [tweet-chunk (<! transfer-chan)]
-        ; (println tweet-chunk)
         (turbine-in tweet-chunk))
       (recur))

-    (statuses-filter :params {:track "trump"}
+    (statuses-filter :params {:track ["trump" "pence"]}
                      :oauth-creds creds
                      :callbacks *callback*)))


modify (defn generate): now requires :model parameter to be supplied

--- src/babel/espanol.cljc
+++ src/babel/espanol.cljc
@@ -54,9 +54,14 @@
 (defn generate
   [spec & {:keys [max-total-depth model truncate-children]
            :or {max-total-depth generate/max-total-depth
-                model (medium)
+                model nil
                 truncate-children true}}]
   (log/debug (str "generating with spec: " (strip-refs spec) " with max-total-depth: " max-total-depth))
+  (if (nil? model)
+    (let [error-message
+          (str "you must supply a model to do generation.")]
+      (log/error error-message)
+      (throw (Exception. error-message))))
   (let [result (generate/generate spec model
                                   :max-total-depth max-total-depth
                                   :truncate-children truncate-children)]


Remove needless anonymous function (thanks, Kibit)

--- src/afterglow/midi.clj
+++ src/afterglow/midi.clj
@@ -255,8 +255,7 @@
   []
   (if (and (mac?) (mmj-installed?))
     (fn [device]
-      (let [mmj-descriptions (set (map :description (filter #(mmj-device? %)
-                                                            (overtone.midi/midi-sources))))]
+      (let [mmj-descriptions (set (map :description (filter mmj-device? (overtone.midi/midi-sources))))]
         (or (mmj-device? device)
             ;; MMJ returns devices whose descriptions match the names of the
             ;; broken devices returned by the broken Java MIDI implementation.


Hack: allow reloading of resources when the 'resources/' dir changes

--- src/blogiseq/utils.clj
+++ src/blogiseq/utils.clj
@@ -10,10 +10,14 @@
                          ~default))
   )

-(defn parse-resource [path]
-  (-> path
-    clojure.java.io/resource
-    slurp)
+(defn- parse-resource
+  "Firstly parse path in the `resources/` dir. If not successful,
+  tries loading this from resources."
+  [path]
+  (if-let [ext-content (slurp (str "resources/" path))]
+    ext-content
+    (slurp (clojure.java.io/resource path))
+    )
   )

 (defn parse-edn-resource [path]


404 on bad response from visual studio online

--- src/build_mon/core.clj
+++ src/build_mon/core.clj
@@ -1,6 +1,7 @@
 (ns build-mon.core
   (:require [ring.adapter.jetty :as ring-jetty]
-            [clj-http.client :as client])
+            [clj-http.client :as client]
+            [cheshire.core :as json])
   (:gen-class))

 (defn generate-html [status result]
@@ -17,18 +18,13 @@
 (defn handler [account project token request]
   (let [last-build-url (str "https://" account  ".visualstudio.com/defaultcollection/"
                             project "/_apis/build/builds?api-version=2.0&$top=1")
-        last-build (-> (client/get last-build-url
-                                   {:accept :json :as :json
-                                    :basic-auth ["USERNAME CAN BE ANY VALUE" token]})
-                       :body :value first)
-        status (:status last-build)
-        result (:result last-build)]
-      (prn "--")
-      (prn "Status:" status)
-      (prn "Result:" result)
+        api-response (client/get last-build-url {:basic-auth ["USERNAME CAN BE ANY VALUE" token]})
+        last-build (try (-> api-response :body (json/parse-string true) :value first)
+                        (catch Exception e))]
+    (when last-build
       {:status 200
        :headers {"Content-Type" "text/html"}
-       :body (generate-html status result)}))
+       :body (generate-html (:status last-build) (:result last-build))})))

 (defn -main [& [vso-account vso-project vso-personal-access-token port]]
   (let [port (Integer. (or port 3000))]


fix bin format when (*config* "project.java_opts") is nil

--- src/cake/tasks/jar.clj
+++ src/cake/tasks/jar.clj
@@ -151,7 +151,7 @@
         (log "Creating standalone executable:" (.getPath binfile))
         (with-open [bin (FileOutputStream. binfile)]
           (let [shebang (format "#!/usr/bin/env sh\nexec java %s -jar $0 \"$@\"\n"
-                                (*config* "project.java_opts"))]
+                                (or (*config* "project.java_opts") ""))]
             (.write bin (.getBytes shebang)))
           (copy uberjar bin))
         (ant Chmod {:file binfile :perm "+x"})))


Moved prog-world higher.

Moved prog-world higher.

--- src/cbfg/world/t1.cljs
+++ src/cbfg/world/t1.cljs
@@ -12,6 +12,12 @@
             [cbfg.lane]))

 ;; TODO: How to assign locations to world entities before rendering?
+
+(def prog-world (atom {})) ; { :world => world-actx
+                           ;   :net-listen-ch => ch
+                           ;   :net-connect-ch => ch
+                           ;   :servers => { server-addr => ports }
+                           ;   :clients => client-info }

 (def run-history
   (atom {:snapshots {0 {}
@@ -50,14 +56,6 @@

 ; -------------------------------------------------------------------

-(def prog-world (atom {})) ; { :world => world-actx
-                           ;   :net-listen-ch => ch
-                           ;   :net-connect-ch => ch
-                           ;   :servers => { server-addr => ports }
-                           ;   :clients => client-info }
-
-; -------------------------------------------------------------------
-
 (defn render-world [app owner]
   (apply dom/ul nil
          (map (fn [[k v]] (dom/li nil (str k ":" v))) app)))
@@ -80,12 +78,11 @@
           (reduce
            (fn [[last-snapshot-ts res] [ts kind args]]
              (if (= kind :snapshot)
-               [ts
-                (conj res
-                      (dom/li #js {:className "snapshot"
-                                   :onMouseEnter #(on-event-focus ts nil)
-                                   :onMouseLeave #(on-event-blur ts nil)}
-                              (str ts)))]
+               [ts (conj res
+                         (dom/li #js {:className "snapshot"
+                                      :onMouseEnter #(on-event-focus ts nil)
+                                      :onMouseLeave #(on-event-blur ts nil)}
+                                 (str ts)))]
                [(or last-snapshot-ts ts)
                 (conj res
                       (dom/li #js {:onMouseEnter


Rename to collsets.

Rename to collsets.

--- src/cbfg/world/t2.cljs
+++ src/cbfg/world/t2.cljs
@@ -20,9 +20,11 @@

 ; --------------------------------------------

+(defn msg-req [actx ch-key m]
+  (act msg-req actx (areq msg-req (ch-key m) m)))
+
 (defn msg-put [actx ch-key m]
-  (act msg-put actx
-       (aput msg-put (ch-key m) m)))
+  (act msg-put actx (aput msg-put (ch-key m) m)))

 (defn msg-put-res [actx ch-key m]
   (let [res-ch (achan actx)]
@@ -33,17 +35,15 @@
                               :status-info [(:op m) :closed :msg-put-res]))))
     res-ch))

-(defn msg-req [actx ch-key m]
-  (act msg-req actx
-       (areq msg-req (ch-key m) m)))
-
-; --------------------------------------------
+; --------------------------------------------
+
+; TODO: Split between logical schema and physical runtime objects.

 (defn make-initial-server-state [actx]
   {:realms {"_system" {:users {"admin" {:pswd "password"}}
-                       :coll-sets {}}
+                       :collsets {}}
             "_lobby" {:users {"_anon" {:pswd ""}}
-                      :coll-sets {}}}})
+                      :collsets {}}}})

 (def initial-lane-state
   {:cur-realm "_lobby" :cur-user-realm "_lobby" :cur-user "_anon"})


Add helper for building ProcessorSuppliers

--- src/cddr/ksml/eval.clj
+++ src/cddr/ksml/eval.clj
@@ -5,6 +5,7 @@
    [clojure.string :as str])
   (:import
    (org.apache.kafka.streams KeyValue)
+   (org.apache.kafka.streams.processor Processor ProcessorSupplier)
    (org.apache.kafka.streams.kstream Predicate
                                      ForeachAction
                                      ValueMapper KeyValueMapper
@@ -78,6 +79,22 @@
     (apply [_ k v]
       (each-fn k v))))

+(defn processor-supplier
+  ([process-fn]
+   (processor-supplier process-fn (constantly nil)))
+
+  ([process-fn punctuate-fn]
+   (reify ProcessorSupplier
+     (get [_]
+       (let [ctx (atom nil)]
+         (reify Processor
+           (init [_ context]
+             (reset! ctx context))
+           (punctuate [_ ts]
+             (punctuate-fn @ctx ts))
+           (process [_ k v]
+             (process-fn @ctx k v))))))))
+
 ;; kstream/ktable ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;
 ;; Lots of methods exist on both KStream and KTable classes. Clojure
@@ -192,26 +209,10 @@
 (defmethod eval-op :process
   ([_ process-fn stream]
    `(.. ~(eval stream)
-        (process (reify ProcessorSupplier
-                   (get [_]
-                     (let [ctx# (atom nil)]
-                       (reify Processor
-                         (init [_ context#]
-                           (reset! ctx# context#))
-                         (process [_ k# v#]
-                           (~process-fn k# v#)))))))))
+        (process (processor-supplier ~process-fn))))
   ([_ process-fn punctuate-fn stream]
    `(.. ~(eval stream)
-        (process (reify ProcessorSupplier
-                   (get [_]
-                     (let [ctx# (atom nil)]
-                       (reify Processor
-                         (init [_ context#]
-                           (reset! ctx# context#))
-                         (punctuate [_ ts#]
-                           (~punctuate-fn @ctx# ts#))
-                         (process [_ k# v#]
-                           (~process-fn @ctx# k# v#))))))))))
+        (process (processor-supplier ~process-fn ~punctuate-fn)))))

 (defmethod eval-op :select-key
   [_ map-fn stream]


use identical? instead of =

use identical? instead of =

--- src/cheshire/parse.clj
+++ src/cheshire/parse.clj
@@ -17,8 +17,8 @@
     `(do
        (.nextToken ~jp)
        (loop [mmap# (transient {})]
-         (if-not (= (.getCurrentToken ~jp)
-                    JsonToken/END_OBJECT)
+         (if-not (identical? (.getCurrentToken ~jp)
+                             JsonToken/END_OBJECT)
            (let [key-str# (.getText ~jp)
                  _# (.nextToken ~jp)
                  key# (~key-fn key-str#)
@@ -36,8 +36,8 @@
        (loop [coll# (transient (if ~array-coerce-fn
                                  (~array-coerce-fn array-field-name#)
                                  []))]
-         (if-not (= (.getCurrentToken ~jp)
-                   JsonToken/END_ARRAY)
+         (if-not (identical? (.getCurrentToken ~jp)
+                             JsonToken/END_ARRAY)
            (let [coll# (conj! coll#
                          (parse* ~jp ~key-fn ~bd? ~array-coerce-fn))]
              (.nextToken ~jp)
@@ -77,7 +77,7 @@
       (str "Cannot parse " (pr-str (.getCurrentToken jp)))))))

 (defn parse [^JsonParser jp key-fn eof array-coerce-fn]
-  (let [key-fn (or (if (= key-fn true) keyword key-fn) identity)]
+  (let [key-fn (or (if (identical? key-fn true) keyword key-fn) identity)]
     (.nextToken jp)
     (condp identical? (.getCurrentToken jp)
       nil


Update examples

Update examples

--- src/clj/inky/entry.clj
+++ src/clj/inky/entry.clj
@@ -193,6 +193,11 @@
   [["Dommy" "0.1.2" "https://github.com/Prismatic/dommy"]
    ["Javelin" "2.4.0" "https://github.com/tailrecursion/javelin"]
    ["core.async" "0.1.267.0-0d7780-alpha" "https://github.com/clojure/core.async"]])
+
+(def previews
+  [["5ac8ba0ce4cb11c3d518d25357a8b11b"
+    "@heyzk"
+    ""]])

 (defn $intro []
   ($layout
@@ -226,29 +231,29 @@
           [:strong "first"]
           " by "
           [:a {:href "https://twitter.com/heyzk"} "@heyzk"]]
-         [:a {:href "/s/dc820ddc03b6e8e1c9ff60572ff873c5"}
-          [:img {:src "http://f.cl.ly/items/06430H393A2M2T1H1N3e/Screen%20Shot%202013-12-19%20at%204.01.24%20PM.png"}]]]
+         [:a {:href "/s/41cbe11f985b27aa8ed947521e776d64"}
+          [:img {:src "http://f.inky.cc.s3.amazonaws.com/top-examples/first.jpg"}]]]
         [:li.sketch-preview
          [:div.sketch-meta
           [:strong "ten.k.processes.redux"]
           " by "
           [:a {:href "https://twitter.com/heyzk"} "@heyzk"]]
          [:a {:href "/s/7e2ed9da90a50773bea5b87b73844669"}
-          [:img {:src "http://s3.amazonaws.com/f.inky.cc/top-examples/7e2ed9da90a50773bea5b87b73844669.png"}]]]
+          [:img {:src "http://f.inky.cc.s3.amazonaws.com/top-examples/tenkredux.jpg"}]]]
         [:li.sketch-preview
          [:div.sketch-meta
           [:strong "instagram.api"]
           " by "
           [:a {:href "https://twitter.com/heyzk"} "@heyzk"]]
          [:a {:href "/s/697d0a5189c1f04e66c09e3968c194f4"}
-          [:img {:src "http://f.cl.ly/items/0h46002N1N1l363d062I/Screen%20Shot%202013-12-20%20at%206.26.41%20PM.png"}]]]
+          [:img {:src "http://f.inky.cc.s3.amazonaws.com/top-examples/instagram.jpg"}]]]
         [:li.sketch-preview
          [:div.sketch-meta
           [:strong "haiku"]
           " by "
           [:a {:href "https://twitter.com/heyzk"} "@heyzk"]]
          [:a {:href "/s/5ac8ba0ce4cb11c3d518d25357a8b11b"}
-          [:img {:src "http://f.cl.ly/items/043q3l0T0q3x052V0D11/Screen%20Shot%202013-12-20%20at%206.24.45%20PM.png"}]]]
+          [:img {:src "http://f.inky.cc.s3.amazonaws.com/top-examples/haiku.jpg"}]]]
         ]]
       [:section.instructions
        [:h3 "How-To"]


PDOK-1599: test temporaly removed

PDOK-1599: test temporaly removed

--- src/test/clojure/pdok/featured/extracts_test.clj
+++ src/test/clojure/pdok/featured/extracts_test.clj
@@ -22,15 +22,6 @@
   (let [[error features-for-extract] (features-for-extract "test" "dummy" "gml2extract" (two-features) "src/test/resources/templates")]
     (is (= 2 )(count features-for-extract))))

-(def test-rendered-feature-gml
-  (let [_ (println (str "feature one " (one-feature)))
-        [error features-for-extract] (features-for-extract "test" "dummy" "gml2extract" (one-feature) "src/test/resources/templates")
-        _ (println (str "error " error))
-        _ (println (str "feature for extracts" features-for-extract))
-        [tiles result-feature] (first features-for-extract)
-        _ (println result-feature)]
-    (is (boolean (re-find #"<geo><gml:Polygon" result-feature)))
-    (is (boolean (re-find #"<naam>PDOK</naam>" result-feature)))))

 (def ^{:private true} extract-type-citygml "citygml")


hosted cards targetting

hosted cards targetting

--- src/clj/game/cards.clj
+++ src/clj/game/cards.clj
@@ -51,7 +51,7 @@

    "Aesops Pawnshop"
    {:abilities [{:msg (msg "trash " (:title target) " and gain 3 [Credits]")
-                 :choices {:req #(= (first (:zone %)) :rig)}
+                 :choices {:req #(and (= (:side %) "Runner") (:installed %))}
                  :effect (effect (gain :credit 3) (trash target))}]}

    "Aggressive Negotiation"
@@ -366,7 +366,7 @@

    "Chop Bot 3000"
    {:abilities [{:msg (msg "trash " (:title target))
-                 :choices {:req #(= (first (:zone %)) :rig)}
+                 :choices {:req #(and (= (:side %) "Runner") (:installed %))}
                  :effect (effect (trash target)
                                  (resolve-ability
                                   {:prompt "Draw 1 card or remove 1 tag" :msg (msg (.toLowerCase target))
@@ -960,10 +960,8 @@

    "Foxfire"
    {:trace {:base 7 :prompt "Choose 1 card to trash" :not-distinct true
-            :choices (req (filter #(or (has? % :subtype "Virtual") (has? % :subtype "Link"))
-                                  (concat (get-in runner [:rig :hardware])
-                                          (get-in runner [:rig :resource])
-                                          (get-in runner [:rig :program]))))
+            :choices {:req #(and (:installed %)
+                                 (or (has? % :subtype "Virtual") (has? % :subtype "Link")))}
             :msg (msg "trash " (:title target)) :effect (effect (trash target))}}

    "Frame Job"
@@ -976,7 +974,8 @@

    "Freelancer"
    {:req (req tagged) :msg (msg "trash " (join ", " (map :title targets)))
-    :choices {:max 2 :req #(= (:zone %) [:rig :resource])} :effect (effect (trash-cards :runner targets))}
+    :choices {:max 2 :req #(and (:installed %) (= (:type %) "Resource"))}
+    :effect (effect (trash-cards :runner targets))}

    "Gabriel Santiago: Consummate Professional"
    {:events {:successful-run {:msg "gain 2 [Credits]" :once :per-turn
@@ -1219,7 +1218,7 @@
    {:events {:runner-turn-begins {:effect (effect (add-prop card :counter 1))}}
     :abilities [{:cost [:click 1]
                  :msg (msg "move " (:counter card) " virus counter to " (:title target))
-                 :choices {:req #(and (= (first (:zone %)) :rig) (has? % :subtype "Virus"))}
+                 :choices {:req #(and (:installed %) (has? % :subtype "Virus"))}
                  :effect (effect (trash card {:cause :ability-cost}) (add-prop target :counter (:counter card)))}]}

    "Indexing"
@@ -1792,6 +1791,7 @@
                                                            :effect (effect (gain :credit 1))} card nil)))))
     :leave-play (req (remove-watch state :order-of-sol))}

+   ;; TODO: take hosted card into account
    "Origami"
    {:effect (effect (gain :max-hand-size
                           (dec (* 2 (count (filter #(= (:title %) "Origami")
@@ -2756,7 +2756,7 @@
    {:advanceable :always}

    "Trade-In"
-   {:prompt "Choose a hardware to trash" :choices (req (get-in runner [:rig :hardware]))
+   {:prompt "Choose a hardware to trash" :choices {:req #(and (:installed %) (= (:type %) "Hardware"))}
     :msg (msg "trash " (:title target) " and gain " (quot (:cost target) 2) " [Credits]")
     :effect (effect (trash target) (gain [:credit (quot (:cost target) 2)])
                     (resolve-ability {:prompt "Choose a Hardware to add to Grip from Stack"
@@ -2831,7 +2831,7 @@
                                   :effect (effect (gain :credit 1))}}}

    "Uninstall"
-   {:choices {:req #(and (= (first (:zone %)) :rig) (#{"Program" "Hardware"} (:type %)))}
+   {:choices {:req #(and (:installed %) (#{"Program" "Hardware"} (:type %)))}
     :msg (msg "move " (:title target) " to his or her grip")
     :effect (effect (move target :hand))}

@@ -3379,7 +3379,8 @@
    "Flare"
    {:abilities [{:prompt "Choose a piece of hardware to trash"
                  :msg (msg "trash " (:title target)) :label "Trash a piece of hardware"
-                 :choices (req (get-in runner [:rig :hardware])) :effect (effect (trash target))}
+                 :choices {:req #(and (:installed %) (:type % "Hardware"))}
+                 :effect (effect (trash target))}
                 {:msg "do 2 meat damage and end the run"
                  :effect (effect (damage :meat 2 {:unpreventable true :card card}) (end-run))}]}

@@ -3680,7 +3681,7 @@

    "Sherlock 1.0"
    {:abilities [{:label "Trace 4 - Add an installed program to the top of Stack"
-                 :trace {:base 4 :choices {:req #(= (:zone %) [:rig :program])}
+                 :trace {:base 4 :choices {:req #(and (:installed %) (= (:type %) "Program"))}
                          :msg (msg "add " (:title target) " to the top of Stack")
                          :effect (effect (move :runner target :deck true))}}]}

@@ -3722,7 +3723,7 @@
    {:abilities [{:msg "do 1 net damage" :effect (effect (damage :net 1 {:card card}))}
                 {:prompt "Choose an AI program to trash" :msg (msg "trashes " (:title target))
                  :label "Trash an AI program" :effect (effect (trash target))
-                 :choices (req (filter #(has? % :subtype "AI") (get-in runner [:rig :program])))}]}
+                 :choices {:req #(and (:installed %) (= (:type %) "Program") (has? % :subtype "AI"))}}]}

    "Taurus"
    {:abilities [{:label "Trace 2 - Trash a piece of hardware"
@@ -3820,6 +3821,7 @@
    "Wotan"
    {:abilities [{:msg "end the run" :effect (effect (end-run))}]}

+   ;; TODO: take hosted installed Fracter into account
    "Wraparound"
    {:abilities [{:msg "end the run" :effect (effect (end-run))}]
     :strength-bonus (req (if (some #(has? % :subtype "Fracter") (get-in runner [:rig :program]))


various cleanup

various cleanup

--- src/clj/ji/service.clj
+++ src/clj/ji/service.clj
@@ -6,12 +6,13 @@
             [clojure.edn :as edn]
             [clojure.core.async :refer [chan go <! >! <!! >!! alt! alts! put! close!]]
             [clojure.core.match :refer [match]]
-            [compojure.core :refer [routes]]
+            [compojure.core :refer [routes GET]]
             [compojure.route :as route])
   (:import [ji.domain.messages GameJoinMessage PlayerSetMessage]))

 (def app
   (routes
+    (GET "/" [] {:status 302 :headers {"Location" "/index.html"} :body ""})
     (route/files "/" {:root "out/public"})
     (route/files "/" {:root "public"})))

@@ -34,7 +35,10 @@
       (game/draw-cards num-add game)
       game)))

-(defn fix-setless-board [game]
+(defn fix-setless-board
+  "If game's board contains sets, returns game as-is, otherwise, adds 3 cards
+  until at least 1 set exists on board"
+  [game]
   (loop [game game]
     (if (empty? (game/solve-board (:board game)))
       (recur (game/draw-cards 3 game))
@@ -72,36 +76,32 @@
   (go
     (loop [game game
            clients []]
-      (println "------------------------------------------")
       (if-not (game-over? game)
         (let [[msg sc] (alts! (cons join-msgs (map :in clients)))]
           (if (= sc join-msgs)
             (let [{:keys [player-id client]} msg
                   clients' (conj clients client)]
-              (println "JOIN MSG" (count clients) msg)
               (recur (-> (game/add-player game player-id)
                          (step-game)
                          (broadcast-game! clients'))
                      clients'))
-            (do (println "GAME MSG" msg sc)
-                (cond
-                  (nil? msg) ;; disconnect player
-                  (let [[client other-clients] (separate-client clients sc)]
-                    (recur (-> game
-                               (game/disconnect-player (:player-id client))
-                               (step-game)
-                               (broadcast-game! other-clients))
-                           other-clients))
+            (do
+              (cond
+                (nil? msg) ;; disconnect player
+                (let [[client other-clients] (separate-client clients sc)]
+                  (recur (-> game
+                             (game/disconnect-player (:player-id client))
+                             (step-game)
+                             (broadcast-game! other-clients))
+                         other-clients))

-                  (satisfies? IGameMessage msg) ;; TODO validate
-                  (recur (-> (apply-message msg game)
-                             (step-game)
-                             (broadcast-game! clients))
-                         clients)
-                  :else
-                  (do (println "UNHANDLED" msg)
-                      (recur game clients))))
-            ))
+                (satisfies? IGameMessage msg) ;; TODO validate
+                (recur (-> (apply-message msg game)
+                           (step-game)
+                           (broadcast-game! clients))
+                       clients)
+                :else
+                (recur game clients)))))
         {:game game :clients clients}))))

 (defn init-game-env!


Update WOFF mime type

Update WOFF mime type

W3C updated WOFF from draft to recommendation status.

--- ring-core/src/ring/util/mime_type.clj
+++ ring-core/src/ring/util/mime_type.clj
@@ -79,7 +79,7 @@
    "txt"   "text/plain"
    "webm"  "video/webm"
    "wmv"   "video/x-ms-wmv"
-   "woff"  "application/x-font-woff"
+   "woff"  "application/font-woff"
    "xbm"   "image/x-xbitmap"
    "xls"   "application/vnd.ms-excel"
    "xml"   "text/xml"


Slightly more comprehensive testing.

Slightly more comprehensive testing.

--- taglines/test/com/lemonodor/commoncrawl_test.clj
+++ taglines/test/com/lemonodor/commoncrawl_test.clj
@@ -48,8 +48,12 @@
         ((c/first-n
           (cc/hfs-arc-item-tap
            (io/file (io/resource "1262847572760_10.arc.gz")))
-          2)
+          5)
          ?url ?arcitem))
     =>
     (produces
-     [["http://1015jamz.com/"] ["http://1015jamz.com/CNet-com-Extras-/3602304"]])))
+     [["http://1015jamz.com/"]
+      ["http://1015jamz.com/CNet-com-Extras-/3602304"]
+      ["http://1015jamz.com/High-School-Invasion/2179263"]
+      ["http://1015jamz.com/Jasmine/4118295"]
+      ["http://1015jamz.com/Suga-Bear/2129607"]])))


return a promise, don't reset! inside the function

--- clojure/src/dnd/db.cljs
+++ clojure/src/dnd/db.cljs
@@ -31,18 +31,13 @@
       (await (levels))
       (.findOneAndUpdate #js {} #js {"$set" #js {:blocks (clj->js @level)}}))))

-(defn load [level]
+(defn load []
   (async
     (-> (await (levels))
       (.findOne #js {})
       await
       (js->clj :keywordize-keys true)
       :blocks
-      (then (fn [blocks]
-              (->> blocks
-                (transform [ALL ALL :color] clj->js)
-                (reset! level))
-              ))
-      )))
+      p/promise)))


upgrade eastwood/lein-ancient

upgrade eastwood/lein-ancient

--- dot/lein/profiles.clj
+++ dot/lein/profiles.clj
@@ -1,8 +1,8 @@
 {:user
  {:plugins [[lein-vanity "0.2.0" :exclusions [org.clojure/clojure]]
-            [lein-ancient "0.6.14"]
+            [lein-ancient "0.6.15"]
             [lein-kibit "0.1.6"]
-            [jonase/eastwood "0.2.5"]
+            [jonase/eastwood "0.2.6"]
             ;;[com.billpiel/sayid "0.0.10"]
             [lein-cljfmt "0.5.7"]]
   :dependencies [[slamhound "1.5.5"]]


info print format fix

info print format fix

--- boot/worker/src/boot/web.clj
+++ boot/worker/src/boot/web.clj
@@ -30,7 +30,7 @@

 (defn spit-web! [webxmlfile serve create destroy]
   (let [xmlfile  (io/file webxmlfile)]
-    (util/info "Creating web.xml...")
+    (util/info "Creating web.xml...\n")
     (spit
       (doto xmlfile io/make-parents)
       (pr-str (web-xml "boot-webapp" "boot-webapp" serve create destroy)))))


Make parsing code more readable and robust

--- src/custard/parser.clj
+++ src/custard/parser.clj
@@ -8,23 +8,22 @@
             [gitiom.tree :as git-tree]
             [me.raynes.fs :as fs]))

-(defn normalize-filename [file]
-  (last (first (re-seq #"(.*)\..*$" file))))
+;;;; Path utilities

-(defn relative-filename [root file]
+(defn strip-extension [path]
+  (last (first (re-seq #"(.*)\..*$" path))))
+
+(defn relative-path [root path]
   (let [root-segments (fs/split root)
-        file-segments (fs/split file)
-        root-size (count root-segments)]
+        path-segments (fs/split path)
+        n (count root-segments)]
     (str/join "/"
-              (condp = (take root-size file-segments)
-                root-segments (drop root-size file-segments)
-                file-segments))))
+              (condp = (take n path-segments)
+                root-segments (drop n path-segments)
+                path-segments))))

-(defn process-filename [[filename data]]
-  (let [name-segments (-> filename
-                          normalize-filename
-                          (str/split #"/"))]
-    [name-segments data]))
+(defn path->segments [path]
+  (fs/split (strip-extension path)))

 (defn recursive-merge [a b]
   (if (and (map? a) (map? b))
@@ -43,7 +42,7 @@
                 (fn [children]
                   (mapv #(process-down % f ctx) children))))))

-(defn normalize-kind [node _]
+(defn set-kind [node _]
   (letfn [(normalize [kind]
             (let [aliases {"project" ["project"]
                            "requirement" ["requirement" "req" "r"]
@@ -59,14 +58,7 @@
              (update node "kind" normalize)
              node)}))

-(defn inject-name [node parent-segments]
-  (let [segments (if (nil? (:name node))
-                   parent-segments
-                   (conj parent-segments (:name node)))]
-    {:node (assoc node :name (str/join "/" segments))
-     :ctx segments}))
-
-(defn inject-parent [node ctx]
+(defn set-parent [node ctx]
   (let [parent ctx]
     {:node (assoc node :parent (or (:parent node)
                                    (:name parent)))
@@ -77,26 +69,27 @@
                  (fn [mapped-here]
                    (mapv #(hash-map :name %) mapped-here)))})

-(defn build-tree [root data]
-  (letfn [(build-node [name-segment data]
+(defn build-tree [data]
+  (letfn [(build-children [name-segments data]
+            (into []
+                  (comp (filter #(map? (second %)))
+                        (map (fn [[k v]]
+                               (build-node (conj name-segments k) v))))
+                  data))
+          (build-node [name-segments data]
             (cond
               (map? data)
-              (into {:name name-segment
-                     :children (->> data
-                                    (map (fn [[k v]]
-                                           (if (map? v)
-                                             (build-node k v))))
-                                    (keep identity)
-                                    (into []))}
+              (into {:name (str/join "/" name-segments)
+                     :children (build-children name-segments data)}
                     (remove #(map? (second %)) data))))
           (build-step [root [name-segment data]]
             (update root :children conj
-                    (build-node name-segment data)))]
-    (let [tree (reduce build-step root data)]
+                    (build-node [name-segment] data)))]
+    (let [root {:name [] :children ()}
+          tree (reduce build-step root data)]
       (-> tree
-          (process-down normalize-kind nil)
-          (process-down inject-name [])
-          (process-down inject-parent nil)
+          (process-down set-kind nil)
+          (process-down set-parent nil)
           (process-down create-mapped-here-links nil)))))

 (defn flatten-tree [node]
@@ -130,24 +123,33 @@
             (build-node graph node))]
     (reduce build-step {:node {} :nodes []} flat-tree)))

-(defn parse-files [files]
-  (let [data (->> files
-                  (map process-filename)
-                  (reduce parse-step {}))
-        tree (build-tree {:children []} data)
+(defn recursive-merge [a b]
+  (if (and (map? a) (map? b))
+    (merge-with recursive-merge a b)
+    (merge a b)))
+
+(defn merge-file-data [m [path data]]
+  (update-in m (path->segments path) recursive-merge data))
+
+(defn process-files [path->data]
+  (let [data (reduce merge-file-data {} path->data)
+        tree (build-tree data)
         flat-tree (flatten-tree tree)
         graph (build-graph flat-tree)]
     graph))

+(defn parse-yaml [data]
+  (try
+    (yaml/parse-string data false)
+    (catch Exception e
+      {:error (str e)})))
+
 (defn parse-uncommitted [dir]
-  (let [files (->> (fs/find-files dir #".*\.yaml$")
-                   (filter #(fs/file? %)))]
-    (parse-files (->> files
-                      (map #(vector (relative-filename dir %)
-                                    (-> %
-                                        slurp
-                                        (yaml/parse-string false))))
-                      (into {})))))
+  (let [files (filter fs/file? (fs/find-files dir #".*\.yaml$"))
+        paths (map #(relative-path dir %) files)
+        datas (map #(parse-yaml (slurp %)) files)
+        path->data (zipmap paths datas)]
+    (process-files path->data)))

 (defn parse-commit [repo commit]
   (let [tree (git-commit/tree repo commit)
@@ -161,6 +163,6 @@
                       data (String. (:data blob))]
                   (assoc res
                          (.getPathString walk)
-                         (yaml/parse-string data false)))
+                         (parse-yaml data)))
                 res))]
-      (parse-files (reduce parse-entry {} lazy-walk)))))
+      (process-files (reduce parse-entry {} lazy-walk)))))


Fix whitespace (#1915)

Fix whitespace (#1915)

Co-authored-by: Scott Crosby <23e8f75489aff47cf2cd5a529e9a769a24e9501c@twosigma.com>

--- scheduler/src/cook/kubernetes/api.clj
+++ scheduler/src/cook/kubernetes/api.clj
@@ -972,172 +972,172 @@
       ; QoS, which requires limits for both memory and cpu
       (.putLimitsItem resources "cpu" (double->quantity cpu-limit)))))

-  (defn add-cook-main-container-to-pod
-    "Create main cook container and add it to the pod"
-    [{:keys [command container pod-supports-cook-init? pod-supports-cook-sidecar? task-id task-request]
-      :or {pod-supports-cook-init? true
-           pod-supports-cook-sidecar? true}}
-     {:keys [computed-mem workdir custom-shell init-container-workdir-volume-mount-fn main-container-checkpoint-volume-mounts main-env-vars
-             pod-name ^V1PodSpec pod-spec pool-name scratch-space-volume-mount-fn sidecar-workdir-volume-mount-fn use-cook-init? volume-mounts]}]
-    (let [{:keys [scalar-requests job resources]} task-request
-          ;; NOTE: The scheduler's adjust-job-resources-for-pool-fn may modify :resources,
-          ;; whereas :scalar-requests always contains the unmodified job resource values.
-          {:strs [cpus]} scalar-requests
-          {:keys [docker]} container
-          {:keys [image port-mapping]} docker
-          ; gpu count is not stored in scalar-requests because Fenzo does not handle gpus in binpacking
-          gpus (or (:gpus resources) 0)
-          gpu-model-requested (constraints/job->gpu-model-requested gpus job pool-name)
-          ;; if disk config for pool has enable-constraint? set to true, add disk info to the job
-          enable-disk-constraint? (regexp-tools/match-based-on-pool-name (config/disk) pool-name :enable-constraint? :default-value false)
-          ;; if user did not specify disk request, use default on pool
-          disk-request (when enable-disk-constraint?
-                         (constraints/job-resources->disk-request resources pool-name))
-          disk-limit (when enable-disk-constraint? (-> resources :disk :limit))
-          ;; if user did not specify disk type, use default on pool
-          disk-type (when enable-disk-constraint? (constraints/job-resources->disk-type resources pool-name))
-          checkpoint (calculate-effective-checkpointing-config job task-id)
-          job-submit-time (tools/job->submit-time job)
-          image (if (synthetic-pod? pod-name)
-                  image
-                  (calculate-effective-image (config/kubernetes) job-submit-time image checkpoint task-id))
-
+(defn add-cook-main-container-to-pod
+  "Create main cook container and add it to the pod"
+  [{:keys [command container pod-supports-cook-init? pod-supports-cook-sidecar? task-id task-request]
+    :or {pod-supports-cook-init? true
+         pod-supports-cook-sidecar? true}}
+   {:keys [computed-mem workdir custom-shell init-container-workdir-volume-mount-fn main-container-checkpoint-volume-mounts main-env-vars
+           pod-name ^V1PodSpec pod-spec pool-name scratch-space-volume-mount-fn sidecar-workdir-volume-mount-fn use-cook-init? volume-mounts]}]
+  (let [{:keys [scalar-requests job resources]} task-request
+        ;; NOTE: The scheduler's adjust-job-resources-for-pool-fn may modify :resources,
+        ;; whereas :scalar-requests always contains the unmodified job resource values.
+        {:strs [cpus]} scalar-requests
+        {:keys [docker]} container
+        {:keys [image port-mapping]} docker
+        ; gpu count is not stored in scalar-requests because Fenzo does not handle gpus in binpacking
+        gpus (or (:gpus resources) 0)
+        gpu-model-requested (constraints/job->gpu-model-requested gpus job pool-name)
+        ;; if disk config for pool has enable-constraint? set to true, add disk info to the job
+        enable-disk-constraint? (regexp-tools/match-based-on-pool-name (config/disk) pool-name :enable-constraint? :default-value false)
+        ;; if user did not specify disk request, use default on pool
+        disk-request (when enable-disk-constraint?
+                       (constraints/job-resources->disk-request resources pool-name))
+        disk-limit (when enable-disk-constraint? (-> resources :disk :limit))
+        ;; if user did not specify disk type, use default on pool
+        disk-type (when enable-disk-constraint? (constraints/job-resources->disk-type resources pool-name))
+        checkpoint (calculate-effective-checkpointing-config job task-id)
+        job-submit-time (tools/job->submit-time job)
+        image (if (synthetic-pod? pod-name)
+                image
+                (calculate-effective-image (config/kubernetes) job-submit-time image checkpoint task-id))
+
+        container (V1Container.)
+        resources (V1ResourceRequirements.)]
+    (.setName container cook-container-name-for-job)
+    (.setCommand container
+                 (conj (or (when use-cook-init? custom-shell) default-shell)
+                       (:value command)))
+    (.setEnv container main-env-vars)
+    (.setImage container image)
+    (doseq [{:keys [container-port host-port protocol]} port-mapping]
+      (when container-port
+        (let [port (V1ContainerPort.)]
+          (.containerPort port (int container-port))
+          (when host-port
+            (.hostPort port (int host-port)))
+          (when protocol
+            (.protocol port protocol))
+          (.addPortsItem container port))))
+
+    ; allocate a TTY to support tools that need to read from stdin
+    (.setTty container true)
+    (.setStdin container true)
+
+    ; Don't add memory limit if user sets job label to allow memory usage above request to "true"
+    (let [allow-memory-usage-above-request (some->> (:memory-limit-job-label-name (config/kubernetes))
+                                                    (get (tools/job-ent->label job))
+                                                    clojure.string/lower-case
+                                                    (= "true"))]
+      (set-mem-cpu-resources resources computed-mem (when-not allow-memory-usage-above-request computed-mem) cpus cpus))
+
+    (when (pos? gpus)
+      (.putLimitsItem resources "nvidia.com/gpu" (double->quantity gpus))
+      (.putRequestsItem resources "nvidia.com/gpu" (double->quantity gpus))
+      (add-node-selector pod-spec "cloud.google.com/gke-accelerator" gpu-model-requested)
+      ; GKE nodes with GPUs have gpu-count label, so synthetic pods need a matching node selector
+      (add-node-selector pod-spec "gpu-count" (-> gpus int str)))
+    (when disk-request
+      ; do not add disk request/limit to synthetic pod yaml because GKE CA will not scale up from 0 if unschedulable pods
+      ; require ephemeral-storage: https://github.com/kubernetes/autoscaler/issues/1869
+      (when-not (synthetic-pod? pod-name)
+        (.putRequestsItem resources "ephemeral-storage" (double->quantity (* constraints/disk-multiplier disk-request)))
+        ; by default, do not set disk-limit
+        (when disk-limit
+          (.putLimitsItem resources "ephemeral-storage" (double->quantity (* constraints/disk-multiplier disk-limit)))))
+      (add-node-selector pod-spec (pool->disk-type-label-name pool-name) disk-type))
+    (.setResources container resources)
+    (.setVolumeMounts container (filterv some? (conj (concat volume-mounts main-container-checkpoint-volume-mounts)
+                                                     (init-container-workdir-volume-mount-fn true)
+                                                     (scratch-space-volume-mount-fn false)
+                                                     (sidecar-workdir-volume-mount-fn true))))
+    (.setWorkingDir container workdir)
+
+    (.addContainersItem pod-spec container)))
+
+(defn add-cook-init-to-pod
+  "Create Cook init container and add it to the pod"
+  [{:keys [computed-mem cpus init-container init-container-checkpoint-volume-mounts init-container-workdir
+           init-container-workdir-volume-mount-fn main-env-vars ^V1PodSpec pod-spec scratch-space-volume-mount-fn sidecar use-cook-sidecar?]}]
+  (when-let [{:keys [command image]} init-container]
+    (let [container (V1Container.)
+          get-resource-requirements-fn (fn [fieldname] (if use-cook-sidecar?
+                                                         (get-in sidecar [:resource-requirements fieldname])
+                                                         0))
+          total-memory-request (add-as-decimals computed-mem (get-resource-requirements-fn :memory-request))
+          total-memory-limit (add-as-decimals computed-mem (get-resource-requirements-fn :memory-limit))
+          total-cpu-request (add-as-decimals cpus (get-resource-requirements-fn :cpu-request))
+          total-cpu-limit (add-as-decimals cpus (get-resource-requirements-fn :cpu-limit))
+          resources (V1ResourceRequirements.)]
+      ; container
+      (.setName container cook-init-container-name)
+      (.setImage container image)
+      (.setCommand container command)
+      (.setWorkingDir container init-container-workdir)
+      (.setEnv container main-env-vars)
+      (.setVolumeMounts container (filterv some? (concat [(init-container-workdir-volume-mount-fn false)
+                                                          (scratch-space-volume-mount-fn false)]
+                                                         init-container-checkpoint-volume-mounts)))
+      (set-mem-cpu-resources resources total-memory-request total-memory-limit total-cpu-request total-cpu-limit)
+      (.setResources container resources)
+      (.addInitContainersItem pod-spec container))))
+
+(defn add-cook-sidecar-to-pod
+  "Create Cook Sidecar container and add it to the pod"
+  [{:keys [main-env-vars ^V1PodSpec pod-spec sidecar sandbox-dir sandbox-volume-mount-fn scratch-space-volume-mount-fn sidecar-workdir sidecar-workdir-volume-mount-fn]}]
+  (when-let [{:keys [command health-check-endpoint image port resource-requirements]} sidecar]
+    (let [{:keys [cpu-request cpu-limit memory-request memory-limit]} resource-requirements
           container (V1Container.)
           resources (V1ResourceRequirements.)]
-      (.setName container cook-container-name-for-job)
-      (.setCommand container
-                   (conj (or (when use-cook-init? custom-shell) default-shell)
-                         (:value command)))
-      (.setEnv container main-env-vars)
+      ; container
+      (.setName container cook-container-name-for-file-server)
       (.setImage container image)
-      (doseq [{:keys [container-port host-port protocol]} port-mapping]
-        (when container-port
-          (let [port (V1ContainerPort.)]
-            (.containerPort port (int container-port))
-            (when host-port
-              (.hostPort port (int host-port)))
-            (when protocol
-              (.protocol port protocol))
-            (.addPortsItem container port))))
-
-      ; allocate a TTY to support tools that need to read from stdin
-      (.setTty container true)
-      (.setStdin container true)
-
-      ; Don't add memory limit if user sets job label to allow memory usage above request to "true"
-      (let [allow-memory-usage-above-request (some->> (:memory-limit-job-label-name (config/kubernetes))
-                                               (get (tools/job-ent->label job))
-                                               clojure.string/lower-case
-                                               (= "true"))]
-        (set-mem-cpu-resources resources computed-mem (when-not allow-memory-usage-above-request computed-mem) cpus cpus))
-
-      (when (pos? gpus)
-        (.putLimitsItem resources "nvidia.com/gpu" (double->quantity gpus))
-        (.putRequestsItem resources "nvidia.com/gpu" (double->quantity gpus))
-        (add-node-selector pod-spec "cloud.google.com/gke-accelerator" gpu-model-requested)
-        ; GKE nodes with GPUs have gpu-count label, so synthetic pods need a matching node selector
-        (add-node-selector pod-spec "gpu-count" (-> gpus int str)))
-      (when disk-request
-        ; do not add disk request/limit to synthetic pod yaml because GKE CA will not scale up from 0 if unschedulable pods
-        ; require ephemeral-storage: https://github.com/kubernetes/autoscaler/issues/1869
-        (when-not (synthetic-pod? pod-name)
-          (.putRequestsItem resources "ephemeral-storage" (double->quantity (* constraints/disk-multiplier disk-request)))
-          ; by default, do not set disk-limit
-          (when disk-limit
-            (.putLimitsItem resources "ephemeral-storage" (double->quantity (* constraints/disk-multiplier disk-limit)))))
-        (add-node-selector pod-spec (pool->disk-type-label-name pool-name) disk-type))
+      (.setCommand container (conj command (str port)))
+      (.setWorkingDir container sidecar-workdir)
+      (.setPorts container [(.containerPort (V1ContainerPort.) (int port))])
+
+      (.setEnv container (conj main-env-vars
+                               ;; DEPRECATED - sidecar should use COOK_SANDBOX instead.
+                               ;; Will remove this environment variable in a future release.
+                               (make-env "COOK_WORKDIR" sandbox-dir)))
+
+      ; optionally enable http-based readiness probe
+      (when health-check-endpoint
+        (let [http-get-action (doto (V1HTTPGetAction.)
+                                (.setPort (-> port int IntOrString.))
+                                (.setPath health-check-endpoint))
+              readiness-probe (doto (V1Probe.)
+                                (.setHttpGet http-get-action))]
+          (.setReadinessProbe container readiness-probe)))
+
+      ; resources
+      (set-mem-cpu-resources resources memory-request memory-limit cpu-request cpu-limit)
       (.setResources container resources)
-      (.setVolumeMounts container (filterv some? (conj (concat volume-mounts main-container-checkpoint-volume-mounts)
-                                                       (init-container-workdir-volume-mount-fn true)
-                                                       (scratch-space-volume-mount-fn false)
-                                                       (sidecar-workdir-volume-mount-fn true))))
-      (.setWorkingDir container workdir)
-
-      (.addContainersItem pod-spec container)))
-
-  (defn add-cook-init-to-pod
-    "Create Cook init container and add it to the pod"
-    [{:keys [computed-mem cpus init-container init-container-checkpoint-volume-mounts init-container-workdir
-             init-container-workdir-volume-mount-fn main-env-vars ^V1PodSpec pod-spec scratch-space-volume-mount-fn sidecar use-cook-sidecar?]}]
-    (when-let [{:keys [command image]} init-container]
-      (let [container (V1Container.)
-            get-resource-requirements-fn (fn [fieldname] (if use-cook-sidecar?
-                                                           (get-in sidecar [:resource-requirements fieldname])
-                                                           0))
-            total-memory-request (add-as-decimals computed-mem (get-resource-requirements-fn :memory-request))
-            total-memory-limit (add-as-decimals computed-mem (get-resource-requirements-fn :memory-limit))
-            total-cpu-request (add-as-decimals cpus (get-resource-requirements-fn :cpu-request))
-            total-cpu-limit (add-as-decimals cpus (get-resource-requirements-fn :cpu-limit))
-            resources (V1ResourceRequirements.)]
-        ; container
-        (.setName container cook-init-container-name)
-        (.setImage container image)
-        (.setCommand container command)
-        (.setWorkingDir container init-container-workdir)
-        (.setEnv container main-env-vars)
-        (.setVolumeMounts container (filterv some? (concat [(init-container-workdir-volume-mount-fn false)
-                                                            (scratch-space-volume-mount-fn false)]
-                                                           init-container-checkpoint-volume-mounts)))
-        (set-mem-cpu-resources resources total-memory-request total-memory-limit total-cpu-request total-cpu-limit)
-        (.setResources container resources)
-        (.addInitContainersItem pod-spec container))))
-
-  (defn add-cook-sidecar-to-pod
-    "Create Cook Sidecar container and add it to the pod"
-    [{:keys [main-env-vars ^V1PodSpec pod-spec sidecar sandbox-dir sandbox-volume-mount-fn scratch-space-volume-mount-fn sidecar-workdir sidecar-workdir-volume-mount-fn]}]
-    (when-let [{:keys [command health-check-endpoint image port resource-requirements]} sidecar]
-      (let [{:keys [cpu-request cpu-limit memory-request memory-limit]} resource-requirements
-            container (V1Container.)
-            resources (V1ResourceRequirements.)]
-        ; container
-        (.setName container cook-container-name-for-file-server)
-        (.setImage container image)
-        (.setCommand container (conj command (str port)))
-        (.setWorkingDir container sidecar-workdir)
-        (.setPorts container [(.containerPort (V1ContainerPort.) (int port))])
-
-        (.setEnv container (conj main-env-vars
-                                 ;; DEPRECATED - sidecar should use COOK_SANDBOX instead.
-                                 ;; Will remove this environment variable in a future release.
-                                 (make-env "COOK_WORKDIR" sandbox-dir)))
-
-        ; optionally enable http-based readiness probe
-        (when health-check-endpoint
-          (let [http-get-action (doto (V1HTTPGetAction.)
-                                  (.setPort (-> port int IntOrString.))
-                                  (.setPath health-check-endpoint))
-                readiness-probe (doto (V1Probe.)
-                                  (.setHttpGet http-get-action))]
-            (.setReadinessProbe container readiness-probe)))
-
-        ; resources
-        (set-mem-cpu-resources resources memory-request memory-limit cpu-request cpu-limit)
-        (.setResources container resources)
-
-        (.setVolumeMounts container [(sandbox-volume-mount-fn true) (sidecar-workdir-volume-mount-fn false) (scratch-space-volume-mount-fn false)])
-        (.addContainersItem pod-spec container))))
-
-  (defn ^V1Pod task-metadata->pod
-    "Given a task-request and other data generate the kubernetes V1Pod to launch that task."
-    [namespace {:keys [cook-pool-taint-name cook-pool-taint-prefix cook-pool-label-name] compute-cluster-name :name}
-     {:keys [task-id command container task-request hostname pod-annotations pod-constraints pod-hostnames-to-avoid
-             pod-labels pod-priority-class pod-supports-cook-init? pod-supports-cook-sidecar?]
-      :or {pod-priority-class cook-job-pod-priority-class
-           pod-supports-cook-init? true
-           pod-supports-cook-sidecar? true}
-      :as task-metadata}]
-    (let [{:keys [scalar-requests job]} task-request
-          ;; NOTE: The scheduler's adjust-job-resources-for-pool-fn may modify :resources,
-          ;; whereas :scalar-requests always contains the unmodified job resource values.
-          {:strs [mem cpus]} scalar-requests
-          {:keys [docker volumes]} container
-          {:keys [parameters]} docker
-          {:keys [environment]} command
-          pool-name (cached-queries/job->pool-name job)
-          pod (V1Pod.)
-          pod-spec (V1PodSpec.)
-          metadata (V1ObjectMeta.)
-          pod-labels (merge (job->pod-labels job) pod-labels)
+
+      (.setVolumeMounts container [(sandbox-volume-mount-fn true) (sidecar-workdir-volume-mount-fn false) (scratch-space-volume-mount-fn false)])
+      (.addContainersItem pod-spec container))))
+
+(defn ^V1Pod task-metadata->pod
+  "Given a task-request and other data generate the kubernetes V1Pod to launch that task."
+  [namespace {:keys [cook-pool-taint-name cook-pool-taint-prefix cook-pool-label-name] compute-cluster-name :name}
+   {:keys [task-id command container task-request hostname pod-annotations pod-constraints pod-hostnames-to-avoid
+           pod-labels pod-priority-class pod-supports-cook-init? pod-supports-cook-sidecar?]
+    :or {pod-priority-class cook-job-pod-priority-class
+         pod-supports-cook-init? true
+         pod-supports-cook-sidecar? true}
+    :as task-metadata}]
+  (let [{:keys [scalar-requests job]} task-request
+        ;; NOTE: The scheduler's adjust-job-resources-for-pool-fn may modify :resources,
+        ;; whereas :scalar-requests always contains the unmodified job resource values.
+        {:strs [mem cpus]} scalar-requests
+        {:keys [docker volumes]} container
+        {:keys [parameters]} docker
+        {:keys [environment]} command
+        pool-name (cached-queries/job->pool-name job)
+        pod (V1Pod.)
+        pod-spec (V1PodSpec.)
+        metadata (V1ObjectMeta.)
+        pod-labels (merge (job->pod-labels job) pod-labels)
           labels (assoc pod-labels cook-pod-label compute-cluster-name)
           security-context (make-security-context parameters (:user command))
           sandbox-dir (:default-workdir (config/kubernetes))
@@ -1225,11 +1225,11 @@
                      (assoc telemetry-version-var-name
                             (or (:application/version application) "undefined")))
           main-env-vars (cond->> (make-filtered-env-vars main-env)
-                          telemetry-agent-host-var-name
-                          (cons (doto
-                                  (V1EnvVar.)
-                                  (.setName telemetry-agent-host-var-name)
-                                  (.valueFrom hostIpEnvVarSource))))
+                                 telemetry-agent-host-var-name
+                                 (cons (doto
+                                         (V1EnvVar.)
+                                         (.setName telemetry-agent-host-var-name)
+                                         (.valueFrom hostIpEnvVarSource))))
           container-attributes {:computed-mem computed-mem
                                 :cpus cpus
                                 :custom-shell custom-shell
@@ -1264,13 +1264,13 @@

       (add-cook-main-container-to-pod task-metadata container-attributes)

-       ;init container
+      ;init container
       (when use-cook-init?
         (add-cook-init-to-pod container-attributes))

       ; sandbox file server container
       (when use-cook-sidecar?
-          (add-cook-sidecar-to-pod container-attributes))
+        (add-cook-sidecar-to-pod container-attributes))

       ; We're either using the hostname (in the case of users' job pods)
       ; or pod-hostnames-to-avoid (in the case of synthetic pods), but not both.


remove verbose logging

remove verbose logging

--- kafka-clj/src/kafka_clj/client.clj
+++ kafka-clj/src/kafka_clj/client.clj
@@ -88,7 +88,7 @@
                                                                      10000)))
                                       30000)]

-    (prn "create-topic-request resp " resp)
+    (info "create-topic-request resp " resp)
     (when-not (#{0 36} (:error_code resp))
       (throw (RuntimeException. (str "Failed to create topic " topic " response " resp))))))

@@ -97,7 +97,7 @@
   [{:keys [state] :as connector} topic]
   (let [topic-auto-create (get-in state [:conf :topic-auto-create])]

-    (prn "try-create-topic : topic-auto-create: " topic-auto-create)
+    (info "try-create-topic : topic-auto-create: " topic-auto-create)

     (if topic-auto-create
       (create-topic-request connector topic)
@@ -298,9 +298,7 @@
   (try

     (if-let [partition-rc (select-partition-rc state topic)]
-      (do
-        (info "send-data partition-rc " partition-rc)
-        (send-data state partition-rc topic (mapv #(assoc % :partition (:id partition-rc)) msgs)))
+      (send-data state partition-rc topic (mapv #(assoc % :partition (:id partition-rc)) msgs))
       (do
         (error-no-partition state topic msgs)))
     (catch Exception e (do