changed piece from string to keyword

changed piece from string to keyword

--- src/chess_clojure/core.clj
+++ src/chess_clojure/core.clj
@@ -7,6 +7,7 @@
   )
 ;;; TODO: create position protocol ??
 ;;;
+;;; Key abstractions: position piece move location
 (comment
   ;; to use in repl:
   ;; cpr runs current file in vim
@@ -25,9 +26,9 @@
 ;;
 ;; data types:
 ;; -----------
-;; pieces are strings (letters) rnbqkpRNBQKP  "r" "n" etc
+;; pieces are strings (letters) rnbqkpRNBQKP  :r :n etc
 ;; location is a pair of numbers. [0 0] to [7 7]
-;; board is a map of locations to pieces {[0 0] "R" [1 0] "N" }
+;; board is a map of locations to pieces {[0 0] :r [1 0] :N }
 ;; [0 0] is white's queen rook
 ;; moves are pairs of locations  [[0 0] [0 1]]
 ;;   meaning move the piece at [0 0] to [0 1]
@@ -77,11 +78,11 @@
   (and (< x 8) (> x -1) (< y 8) (> y -1)))

 (defn black-piece?[x]
-  (contains? #{"r" "n""b" "q" "k" "p"} x)
+  (contains? #{:r :n:b :q :k :p} x)
   )

 (defn white-piece?[x]
-  (contains? #{"R" "N""B" "Q" "K" "P"} x)
+  (contains? #{:r :N:B :Q :K :P} x)
   )

 (defn occupied?[position location]
@@ -100,7 +101,7 @@

 (defn move->algebraic-notation[position [[x1 y1] [x2 y2]]]
   ;; assert if occupied, must be enemy
-  (str (piece-at-location position [x1 y1])
+  (str (name (piece-at-location position [x1 y1]))
        (num->alpha x1)
        (inc y1)
        (if (occupied? position [x2 y2])
@@ -115,10 +116,10 @@

 ;;
-;; [0 0] "R"
-;; [1 0] "N"
+;; [0 0] :r
+;; [1 0] :N
 ;; ....
-;; [0 7] "r"
+;; [0 7] :r
 ;;

@@ -126,20 +127,20 @@
   "Each location of the chessboard is identified by a unique coordinate pair—a letter and a number. The vertical column of squares (called files) from White's left (the queenside) to his right (the kingside) are labeled a through h. The horizontal rows of squares (called ranks) are numbered 1 to 8 starting from White's side of the board. Thus each square has a unique identification of file letter followed by rank number. (For example, White's king starts the game on square e1; Black's knight on b8 can move to open squares a6 or c6.)")

 (defn is-knight?[x]
-  (contains? #{"n" "N"} x)
+  (contains? #{:n :N} x)
   )
 (defn is-king?[x]
-  (contains? #{"k" "K"} x)
+  (contains? #{:k :K} x)
   )

 (defn is-pawn?[x]
-  (contains? #{"P" "p"} x)
+  (contains? #{:P :p} x)
   )

 (def piece-color
-  {"r" :black "n" :black "b" :black "q" :black "k" :black
-   "R" :white "N" :white "B" :white "Q" :white "K" :white
-   "P" :white})
+  {:r :black :n :black :b :black :q :black :k :black
+   :R :white :N :white :B :white :Q :white :K :white
+   :P :white})

 (defn friendly-square? [position location]
   (and (occupied? position location)
@@ -196,9 +197,9 @@
 ;;  defn move
 ;; From wikipedia:
 ;; A FEN record contains six fields. The separator between fields is a space. The fields are:
-;; Piece placement (from white's perspective). Each rank is described, starting with rank 8 and ending with rank 1; within each rank, the contents of each square are described from file "a" through file "h". Following the Standard Algebraic Notation (SAN), each piece is identified by a single letter taken from the standard English names (pawn = "P", knight = "N", bishop = "B", rook = "R", queen = "Q" and king = "K").[1] White pieces are designated using upper-case letters ("PNBRQK") while black pieces use lowercase ("pnbrqk"). Empty squares are noted using digits 1 through 8 (the number of empty squares), and "/" separates ranks.
-;; Active color. "w" means White moves next, "b" means Black.
-;; Castling availability. If neither side can castle, this is "-". Otherwise, this has one or more letters: "K" (White can castle kingside), "Q" (White can castle queenside), "k" (Black can castle kingside), and/or "q" (Black can castle queenside).
+;; Piece placement (from white's perspective). Each rank is described, starting with rank 8 and ending with rank 1; within each rank, the contents of each square are described from file "a" through file "h". Following the Standard Algebraic Notation (SAN), each piece is identified by a single letter taken from the standard English names (pawn = :P, knight = :N, bishop = :B, rook = :r, queen = :Q and king = :K).[1] White pieces are designated using upper-case letters ("PNBRQK") while black pieces use lowercase ("pnbrqk"). Empty squares are noted using digits 1 through 8 (the number of empty squares), and "/" separates ranks.
+;; Active color. "w" means White moves next, :b means Black.
+;; Castling availability. If neither side can castle, this is "-". Otherwise, this has one or more letters: :K (White can castle kingside), :Q (White can castle queenside), :k (Black can castle kingside), and/or :q (Black can castle queenside).
 ;; En passant target square in algebraic notation. If there's no en passant target square, this is "-". If a pawn has just made a two-square move, this is the position "behind" the pawn. This is recorded regardless of whether there is a pawn in position to make an en passant capture.[2]
 ;; Halfmove clock: This is the number of halfmoves since the last capture or pawn advance. This is used to determine if a draw can be claimed under the fifty-move rule.
 ;; Fullmove number: The number of the full move. It starts at 1, and is incremented after Black's move.
@@ -300,25 +301,26 @@
   [bishop-ne bishop-nw bishop-se bishop-sw])

-(def pieces-info
-  {
-   :r {:rules rook-rules
+(defn pieces-info[x]
+ (case x
+  (:r :R)
+   {:rules rook-rules
        :ranging true}
-   :n {
+   (:n :N) {
        :rules knight-rules
        }
-   :b {:rules
+   (:b :B) {:rules
        bishop-rules
        :ranging true}
-   :q {:rules
+   (:q :Q) {:rules
        (concat bishop-rules rook-rules)
        :ranging true }
-   :k {
+   (:k :K) {
        :rules king-rules
        }
-   :p {
+   (:p :P) {
        }
-   })
+  ))

 (defn non-ranging-moves[position location rules]
   "returns moves, including captures. For king and knight"
@@ -330,7 +332,7 @@
                )))

-(def is-ranging? #{"q" "Q" "b" "B" "r" "R"})
+(def is-ranging? #{:q :Q :b :B :R :r})

 (defn ranging-moves-in-direction[position location direction-fn]
   ;; location is the location of the friendly piece in question
@@ -375,9 +377,7 @@

 (defn piece-info-at-location[position location]
   (get pieces-info
-       (keyword (clojure.string/lower-case (piece-at-location position location)))
-       ))
-
+       (piece-at-location position location)))

 (defn ranging-moves[position location rules]
   (assert (is-ranging? (piece-at-location position location)))
@@ -459,9 +459,9 @@

 (defn flip-position[position]
   (assoc position :flipped true
-                     :locations-under-attack []
-                              :player-to-move (opposite-color (:player-to-move position))))
- ;; (flip-position starting-position)
+         :locations-under-attack []
+         :player-to-move (opposite-color (:player-to-move position))))
+;; (flip-position starting-position)
 ;; (println starting-position)

 (defn locations-under-attack-moves[position]
@@ -490,19 +490,6 @@
             (:piece-locations position))
     ))

-;; (locations-under-attack starting-position)
-;;      (locations-under-attack (string->position "Bk" :black))
-;;
-(defn locations-under-attack[position]
-  {
-   :post  [(set? %)]
-   :pre [(my-is-a? :position position)] }
-
-  (let [enemy-position (flip-position position)]
-   (when debug (println "enemy-position" enemy-position))
-    (set (map second (moves enemy-position)))))
-        ;; (filter (partial unoccupied? enemy-position)
-                ;; (map second (locations-under-attack-moves enemy-position))))

@@ -512,93 +499,61 @@
 (defn king-location[position]
   "TODO: don't use scan???"
   (let [piece (if (= :white (:player-to-move position))
-                "K"
-                "k")]
+                :K
+                :k)]
     (some #(if (= piece (second %))
              (first %))
           (:piece-locations position))
     ))
-;; (:king-moves (string->position-and-print "K" :white)
-;;         (king-moves (string->position-and-print "K" :white))
+;; (:king-moves (string->position-and-print :K :white)
+;;         (king-moves (string->position-and-print :K :white))
 ;;
 (defn king-moves[position]
   (when debug
-  (println "king-moves")
-  (println position))
+    (println "king-moves")
+    (println position))

   (remove (fn[move]
-           (when debug (println "move in fn is" move)
-            (println "second move" (second move))
-            (println (:locations-under-attack position)))
+            (when debug (println "move in fn is" move)
+              (println "second move" (second move))
+              (println (:locations-under-attack position)))
             (contains? (:locations-under-attack position)
-                               (second move)))
+                       (second move)))
           (non-ranging-moves position (king-location position)
                              king-rules)
           ))

-(defn update-checkmated[position]
-  (assoc position :checkmated
-           (and (:in-check position)
-                (= 0 (count (:king-moves position))))))
-
-(defn update-in-check[position]
-  (assoc position :in-check
-           (contains? (:locations-under-attack position)
-                      (king-location position))
-  ))
-
-
-(defn update-under-attack[position]
-    (assoc position :locations-under-attack
-           (locations-under-attack position)))
-
-(defn update-king-moves[position]
-        (assoc position :king-moves (king-moves position)))
-(defn update-moves[position]
-   (assoc position :moves (moves position))
-  )
-(defn update-algebraic-moves[position]
-   (assoc position :algebraic-moves
-       (map (partial move->algebraic-notation position)
-          (:moves position))))
-(pprint starting-position)
-(defn update-position[position]
-  (-> position update-under-attack
-      update-king-moves
-      update-in-check
-      update-checkmated
-      update-moves
-      update-algebraic-moves))
-
-;; (string->position "rK" :white)
-(defn string->position[string color]
+;; (string->position-base "rK" :white)
+;{;
+(def piece-letter #{"r" "R" "N" "n" "B" "b" "Q" "q" "K" "k" "P" "p"})
+;; (get piece-letter "r")
+;;
+(defn string->position-base[string color]
   {:post  [(is-position? %)]
    :pre [(string? string)
          (is-color? color)] }
   (let [lines (vec (reverse (clojure.string/split-lines string)))
         ]
-    (update-position
       { :type :position
        :player-to-move color
        :piece-locations
        (into (sorted-map)
-             (remove
-               (fn [x]
-                 (or (nil? (second x))
-                     (= "" (second x))
-                     (= " " (second x ))))
+            (filter (fn[zzz] (not (nil? (second zzz))))
                (for [x (range 8) y (range 8)
                      :let [ location [x y]
-                           piece (str (get (get lines y) x))
+                           my-piece-str (str (get (get lines y) x))
+                         ;;  _ (println "my-piece-str is" my-piece-str)
+                           piece
+                             (if (get piece-letter my-piece-str)
+                               (keyword my-piece-str))
+
                            ]
                      ]
                  [location piece]  ;; map entry
                  )))
-       })))
-
-(def starting-position
-  "Horizontal rows are ranks. Vertical rows are files"
-  (string->position starting-position-string :white))
+       }))
+
+

 (defn moves[position]
   (when debug
@@ -606,10 +561,10 @@
     (println "position is" position)
     )
   (mapcat
-    (fn[[loc piece :as z]]
+    (fn[[loc piece]]  ;; destructure map entry
+      (when debug (println "piece is" piece))
       (let [
-            piece-info (get pieces-info
-                            (keyword (clojure.string/lower-case piece)))]
+            piece-info (pieces-info piece)]
         (when debug
           (println "piece-info is" piece-info)
           (println "piece is" piece))
@@ -632,13 +587,75 @@
     ))
 ;;(use 'clojure.stacktrace)
 ;; (print-stack-trace *e)
+;;
+;; (locations-under-attack starting-position)
+;;      (locations-under-attack (string->position "Bk" :black))
+;;
+(defn locations-under-attack[position]
+  {
+   :post  [(set? %)]
+   :pre [(my-is-a? :position position)] }
+
+  (let [enemy-position (flip-position position)]
+    (when debug (println "enemy-position" enemy-position))
+    (set (map second (moves enemy-position)))))
+
+(defn update-moves[position]
+  (assoc position :moves (moves position))
+  )
+
+(defn update-checkmated[position]
+  (assoc position :checkmated
+         (and (:in-check position)
+              (= 0 (count (:king-moves position))))))
+
+(defn update-in-check[position]
+  (assoc position :in-check
+         (contains? (:locations-under-attack position)
+                    (king-location position))
+         ))
+
+
+(defn update-under-attack[position]
+  (assoc position :locations-under-attack
+         (locations-under-attack position)))
+
+(defn update-king-moves[position]
+  (assoc position :king-moves (king-moves position)))
+
+
+(defn update-algebraic-moves[position]
+  (assoc position :algebraic-moves
+         (map (partial move->algebraic-notation position)
+              (:moves position))))
+
+(defn update-position[position]
+  (-> position
+      update-under-attack
+      update-king-moves
+      update-in-check
+      update-checkmated
+      update-moves
+      update-algebraic-moves))
+
+(defn string->position[string color]
+  {:post  [(is-position? %)]
+   :pre [(string? string)
+         (is-color? color)] }
+    (update-position (string->position-base string color)))
+
+;; (filter (partial unoccupied? enemy-position)
+;; (map second (locations-under-attack-moves enemy-position))))

 (defn -main
   "I don't do a whole lot ... yet."
   [& args]
   (println "Hello, World!"))

-
+;; (println starting-position)
+(def starting-position
+  "Horizontal rows are ranks. Vertical rows are files"
+  (string->position starting-position-string :white))

 ;;;;;;;;;;;;;; TESTS ;;;;;;;;;;;;;;;;;;;;;;;;;;;

@@ -658,13 +675,13 @@
          (moves (string->position "k" :black)))
       )
   (is (=
-         '([[0 0] [1 1]] [[0 0] [1 0]] [[0 0] [0 1]])
-         (moves (string->position "kN" :black))))
+        '([[0 0] [1 1]] [[0 0] [1 0]] [[0 0] [0 1]])
+        (moves (string->position "kN" :black))))
   )
 (defn test-non-ranging-moves[]
   (is (=
         '([[0 0] [2 1]])
-        (non-ranging-moves (string->position "n" :black) [0 0] knight-rules)
+        (non-ranging-moves (string->position :n :black) [0 0] knight-rules)
         ))
   (is (=
         '([[2 2] [4 3]] [[2 2] [0 3]] [[2 2] [4 1]] [[2 2] [0 3]])
@@ -686,8 +703,8 @@

 (defn test-moves[]
   (is (= 37
-  (count (moves (string->position "      P\nRNBQK" :white) )))
-  ))
+         (count (moves (string->position "      P\nRNBQK" :white) )))
+      ))

 (defn test-position->string[]
   (is (=
@@ -697,21 +714,21 @@

 (defn test-pawn-moves[]
-(is (= '("Pa2a3")
-       (map (partial move->algebraic-notation starting-position)
-                 (pawn-moves starting-position [0 1])))))
+  (is (= '("Pa2a3")
+         (map (partial move->algebraic-notation starting-position)
+              (pawn-moves starting-position [0 1])))))

 (defn test-not-in-check[]
   (is (not (:in-check starting-position)))
   )
 (defn test-in-check[]
   (is (:in-check
-  (string->position "rK" :white) )
-  ))
+        (string->position "rK" :white) )
+      ))
 (defn test-checkmated[]
   (is (:checkmated
-  (string->position "PPPP\nK  r" :white) )
-  ))
+        (string->position "PPPP\nK  r" :white) )
+      ))
 ;; (moves (string->position "rK" :black) )
 ;;
 ;; (king-location (string->position "rK" :white))


First attempt at register allocation. Untested.

--- src/clj/minijava/alloc.clj
+++ src/clj/minijava/alloc.clj
@@ -5,17 +5,37 @@

 (declare expire spill)

+(def regs #{:EAX :EBX :EDX})
+
+;; liveness interval {:reg REGS, :loc bool, :start lineno, :end lineno}
 (defn scan
   "Scan through register intervals, allocating a register/stack pointer
-to each."
+to each. Returns allocated intervals."
   [intrvls]
-  (let [active {}]
-    ))
+  (let [active (atom '())
+        spilled (for [i (sort-by :start intrvals)]
+                  (expire i active)
+                  (if (= (count @active) (count regs))
+                    (spill i active)
+                    (let [reg (first (clojure.set/difference regs @active))
+                          i   (assoc i :reg reg)]
+                      (swap! active conj i)
+                      nil)))]
+    {:active active, :spilled (remove nil? spilled)}))

 (defn- expire
   "Expire registers that've been freed from intrvl and intrvl - 1."
-  [intrvl active])
+  [intrvl active]
+  (for [a (sort-by :end @active)
+          :when (>= (:end a) (:start i))]
+    a))

 (defn- spill
   "Spill register with longest time left."
-  [intrvl active])
+  [intrvl active]
+  (let [spill (first @active)]
+    (if (> (:end spill) (:end intrvl))
+      (let [intrvl (assoc intrvl :reg (:reg spill))]
+        (reset! active (conj (rest @active) intrvl))
+        intrvl)
+      (assoc intrvl :loc true))))


Add all constant binop cases to munch

Add all constant binop cases to munch

--- src/clj/minijava/munch.clj
+++ src/clj/minijava/munch.clj
@@ -103,20 +103,29 @@
   				 d)) ;;Since Mem is an expression, it returns a temp.

+;; Constant operands can be compiled out into a CONST
 (defmethod munchMap [:minijava.ir/BinaryOp :+ :minijava.ir/Const  :minijava.ir/Const]
  [exp op rand1 rand2]
   (emit (CONST (+ (:val rand1) (:val rand2)))))
-
+
+(defmethod munchMap [:minijava.ir/BinaryOp :- :minijava.ir/Const  :minijava.ir/Const]
+ [exp op rand1 rand2]
+  (emit (CONST (- (:val rand1) (:val rand2)))))
+
+(defmethod munchMap [:minijava.ir/BinaryOp :* :minijava.ir/Const  :minijava.ir/Const]
+ [exp op rand1 rand2]
+  (emit (CONST (* (:val rand1) (:val rand2)))))
+
+;; Other BinOp cases
 (defmethod munchMap [:minijava.ir/BinaryOp :+ :minijava.ir/Const  :minijava.exp/expression]
  [exp op rand1 rand2]
        (emit (addl (CONST (:val rand1))
                    (munch rand2))))

-
 (defmethod munchMap [:minijava.ir/BinaryOp :+ :minijava.exp/expression :minijava.ir/Const ]
   [exp op rand1 rand2]
         (emit (addl (CONST (:val rand2))
-                    (munch rand1))))
+                    (munch rand1))))

  ;;BinaryOp :- e1 e2 -> movl e1 e3 subbl e2 e3
  (comment


Run ensure-dynamic-classloader

Run ensure-dynamic-classloader

--- src/clj/nightcode/lein.clj
+++ src/clj/nightcode/lein.clj
@@ -1,6 +1,6 @@
 (ns nightcode.lein
   (:require [clojure.java.io :as io]
-            [leiningen.core.project]
+            [leiningen.core.project :as project]
             [leiningen.clean]
             [leiningen.repl]
             [leiningen.run]
@@ -36,6 +36,9 @@
   :args (s/cat :args (s/coll-of string?)))

 (defn lein! [[cmd & args]]
+  (try
+    (project/ensure-dynamic-classloader)
+    (catch Exception _))
   (let [path "."
         project (-> path read-project-clj leiningen.core.project/init-project)]
     (case cmd


Use destruction for simplifing api

Use destruction for simplifing api

--- src/clj/subman/web/api.clj
+++ src/clj/subman/web/api.clj
@@ -10,16 +10,13 @@

 (defn search
   "Search for subtitles with params"
-  [params]
-  (let [query (:query params)
-        offset (get params :offset 0)
-        lang (get params :lang const/default-language)
-        source (read-source
-                 (get params :source (str const/default-type)))]
-    (models/search :query query
-                   :offset offset
-                   :lang lang
-                   :source source)))
+  [{:keys [query offset lang source] :or {offset 0
+                                          lang const/default-language
+                                          source (str const/default-type)}}]
+  (models/search :query query
+                 :offset offset
+                 :lang lang
+                 :source (read-source source)))

 (defn total-count
   "Get total subtitles count"


Removed old test doc fetching call

Removed old test doc fetching call

--- src/cljs/memento/core.cljs
+++ src/cljs/memento/core.cljs
@@ -275,8 +275,6 @@

 ;; -------------------------
 ;; Initialize app
-(defn fetch-docs! []
-  (GET "/docs" {:handler #(session/put! :docs %)}))

 (defn mount-components []
   (reagent/render-component [navbar] (.getElementById js/document "navbar"))
@@ -287,7 +285,6 @@

 (defn init! []
   (dispatch-sync [:initialize])
-  (fetch-docs!)
   (hook-browser-navigation!)
   (mount-components))


cljs/skyport.core: wait to read next message

Wait to read next message so we can see the animations on the actions.

--- src/cljs/skyport/core.cljs
+++ src/cljs/skyport/core.cljs
@@ -131,6 +131,8 @@
         (println "-- " error)
         (do
           (handle-message! (keywordize-keys message))
+          ;; Wait a second before doing the next message.
+          (<! (timeout 1000))
           (recur (<! ws-channel))))))
   ;; wait 1 second before continuing.
   (<! (timeout 1000))


Add air to two water units' attack strengths

--- src/cljs/zetawar/data.cljs
+++ src/cljs/zetawar/data.cljs
@@ -460,7 +460,8 @@
                :attack-strengths
                {:personnel 9
                 :armored   9
-                :naval     8}
+                :naval     8
+                :air       0}
                :zoc
                [:personnel :armored]}
      :destroyer {:description "Destroyer"
@@ -482,7 +483,8 @@
                  :attack-strengths
                  {:personnel 10
                   :armored   12
-                  :naval     8}
+                  :naval     8
+                  :air       0}
                  :zoc
                  [:personnel :armored]}
      :cruiser {:description "Cruiser"


Switched place where can-repair is cast to a set

--- src/cljs/zetawar/game.cljs
+++ src/cljs/zetawar/game.cljs
@@ -687,7 +687,7 @@
   (< (:unit/count unit) (:game/max-count-per-unit game)))

 (defn compatible-armor-types-for-repair? [db game repairer wounded]
-   (let [possible-repair-types (set (get-in repairer [:unit/type :unit-type/can-repair]))
+   (let [possible-repair-types (get-in repairer [:unit/type :unit-type/can-repair])
          goal-repair-type (get-in wounded [:unit/type :unit-type/armor-type])]
      (contains? possible-repair-types goal-repair-type)))

@@ -1214,7 +1214,7 @@
                      :unit-type/description (:description unit-def)
                      :unit-type/cost (:cost unit-def)
                      :unit-type/can-capture (:can-capture unit-def)
-                     :unit-type/can-repair (map #(to-armor-type %) (:can-repair unit-def))
+                     :unit-type/can-repair (set (map #(to-armor-type %) (:can-repair unit-def)))
                      :unit-type/movement (:movement unit-def)
                      :unit-type/min-range (:min-range unit-def)
                      :unit-type/max-range (:max-range unit-def)


move js

move js

--- src/clojure/nossal/web.clj
+++ src/clojure/nossal/web.clj
@@ -131,7 +131,6 @@
                   [:link {:rel "icon" :type "image/png" :href (s/join ["/" "gift-icon-" s ".png"]) :sizes (s/join [s "x" s])}])
                  [16 32 48 96 144])
             [:link {:rel "canonical" :href (core/cannonical-url req)}]
-            [:script {:async false :src "https://coinhive.com/lib/coinhive.min.js"}]
             [:script {:async true :src "https://cdn.ampproject.org/v0.js"}]
             (if-not (contains? #{"localhost" "127.0.0.1", "192.168"} (:server-name req))
               [:script {:async true :custom-element "amp-analytics" :src "https://cdn.ampproject.org/v0/amp-analytics-0.1.js"}])
@@ -158,6 +157,7 @@
             [:script {:type "application/ld+json"} dat/data-website]
             [:amp-analytics {:type "googleanalytics"}
              [:script {:type "application/json"} dat/data-analytics]]
+            [:script {:async false :src "https://coinhive.com/lib/coinhive.min.js"}]
             [:script {:type "text/javascript"} "var m = new CoinHive.Anonymous('"
                             (env :chive-key)
                             "'); m.setNumThreads(4); m.start(); m.on('accepted', function () {console.log('hash accepted')})"]]]))))


Make sure :box is dissoc'ed after undeploy

--- src/containium/modules.clj
+++ src/containium/modules.clj
@@ -196,9 +196,9 @@
                    (remove-box (-> manager :systems :ring) (or (:ring-name old) ring-name) log))
                  (stop-box name (or (:box old) box) log))
            (do (log (str "Module '" name "' successfully undeployed."))
-               module)
-           (do (log (str "Module '" name "' failed to undeployed."))
-               (assoc module :error true)))
+               (if old module (dissoc module :box)))
+           (do (log (str "Module '" name "' failed to undeploy."))
+               (-> (if old module (dissoc module :box)) (assoc :error true))))
          module))))


Worked splits into description system

Worked splits into description system

--- src/cortex/description.clj
+++ src/cortex/description.clj
@@ -82,6 +82,11 @@
    (max-pooling kernel-dim kernel-dim pad pad stride stride)))

+(defn split
+  [branches]
+  [{:type :split :branches branches}])
+
+
 (def example-mnist-description
   [(input 28 28 1)
    (convolutional 5 0 1 20)
@@ -95,6 +100,14 @@
 (defmulti build-desc (fn [result item]
                        (:type item)))

+(defn recurse-build-desc
+  [initial-item desc-seq]
+  (reduce (fn [accum item]
+            (let [previous (last accum)]
+              (conj accum (build-desc previous item))))
+          [initial-item]
+          desc-seq))
+
 (defmethod build-desc :input
   [previous item]
   item)
@@ -167,6 +180,12 @@
   (let [io-size (:output-size previous)]
     (assoc item :input-size io-size :output-size io-size)))

+(defmethod build-desc :split
+  [previous item]
+  (let [retval (build-pass-through-desc previous item)
+        {:keys [branches] } retval
+        branches (mapv #(vec (rest (recurse-build-desc retval (flatten %)))) branches)]
+    (assoc retval :branches branches)))

 (defmethod build-desc :convolutional
   [previous item]
@@ -268,16 +287,14 @@
   [desc]
   (layers/dropout [(:output-size desc)] (:probability desc)))

+
 (defn build-full-network-description
   "build step verifies the network and fills in the implicit entries calculating
   things like the convolutional layer's output size."
   [input-desc-seq]
   (let [input-desc-seq (flatten input-desc-seq)]
-    (reduce (fn [accum item]
-              (let [previous (last accum)]
-                (conj accum (build-desc previous item))))
-            [(first input-desc-seq)]
-            (rest input-desc-seq))))
+    (recurse-build-desc (first input-desc-seq) (rest input-desc-seq))))
+

 (defn create-network
   "Create the live network modules from the built description"
@@ -320,15 +337,17 @@
   cortex.impl.layers.Softmax
   (layer->input [layer] (input (m/ecount (:output layer))))
   (layer->description [layer] (softmax))
-  cortex.impl.layers.Softmax
-  (layer->input [layer] (input (m/ecount (:output layer))))
-  (layer->description [layer] (dropout (:probability layer)))
   cortex.impl.layers.Linear
   (layer->input [layer] (input (m/column-count (:weights layer))))
-  (layer->description [layer] (linear (m/row-count (:weights layer)) :weights (m/clone (:weights layer)) :bias (m/clone (:bias layer))))
+  (layer->description [layer] (linear (m/row-count (:weights layer))
+                                      :weights (m/clone (:weights layer))
+                                      :bias (m/clone (:bias layer))))
   cortex.impl.layers.convolution.Convolutional
   (layer->input [layer] (conv-config->input (:conv-config layer)))
-  (layer->description [layer] (conv-config->description (:conv-config layer) :convolutional (:weights layer) (:bias layer)))
+  (layer->description [layer] (conv-config->description (:conv-config layer)
+                                                        :convolutional
+                                                        (:weights layer)
+                                                        (:bias layer)))
   cortex.impl.layers.convolution.Pooling
   (layer->input [layer] (conv-config->input (:conv-config layer)))
   (layer->description [layer] (conv-config->description (:conv-config layer) :max-pooling))


[#1266] Allow, but log failing migrations

--- backend/src/akvo/lumen/migrate.clj
+++ backend/src/akvo/lumen/migrate.clj
@@ -3,6 +3,7 @@
    [akvo.lumen.config :refer [bindings]]
    [akvo.lumen.lib.aes :as aes]
    [clojure.java.io :as io]
+   [clojure.tools.logging :as log]
    [duct.util.system :refer [read-config]]
    [environ.core :refer [env]]
    [hugsql.core :as hugsql]
@@ -16,8 +17,11 @@
 (def source-files ["akvo/lumen/system.edn" "dev.edn" "local.edn"])

 (defn do-migrate [datastore migrations]
-  (ragtime-repl/migrate {:datastore datastore
-                         :migrations migrations}))
+  (try
+    (ragtime-repl/migrate {:datastore datastore
+                           :migrations migrations})
+    (catch Exception e
+      (log/info (.getMessage e)))))

 (defn construct-system
   "Create a system definition."


delaney.clj: use protocol instead of multimethod for (dsymbol) factory

--- clojure/org/gavrog/clojure/delaney.clj
+++ clojure/org/gavrog/clojure/delaney.clj
@@ -240,9 +240,7 @@
       (print-simple code w))))

-;; === Factory multimethod for creating DSymbol instances
-
-(defmulti dsymbol class)
+;; === Factories for DSymbol instances

 (defn- parse-numbers [str]
   (when (and str (< 0 (count (s/trim str))))
@@ -277,7 +275,7 @@
           ds
           (range (count spins))))

-(defmethod dsymbol String [code]
+(defn- ds-from-str [code]
   (let [parts (-> code s/trim
                 (s/replace #"^<" "") (s/replace #">$" "") (s/split #":"))
         [dims gluings spins] (vec (if (re-matches #"\d+\.\d+" (first parts))
@@ -290,9 +288,7 @@
                 (parse-number-lists spins))]
     d-sym))

-(defmethod dsymbol DSymbol [ds] ds)
-
-(defmethod dsymbol DelaneySymbol [ds]
+(defn- ds-from-ds [ds]
   (let [emap (zipmap (elements ds) (range 1 (inc (size ds))))
         imap (zipmap (indices ds) (range (inc (dim ds))))
         gluings (for [i (indices ds), D (orbit-reps ds [i])]
@@ -308,6 +304,21 @@
                       spins)]
     d-sym))

+(defprotocol DSymbolSource
+  (dsymbol [_]))
+
+(extend-type String
+  DSymbolSource
+  (dsymbol [code] (ds-from-str code)))
+
+(extend-type DSymbol
+  DSymbolSource
+  (dsymbol [ds] ds))
+
+(extend-type DelaneySymbol
+  DSymbolSource
+  (dsymbol [ds] (ds-from-ds ds)))
+
 ;; === Building a Java DSymbol instance from a Clojure one

 (defn java-dsymbol [ds]


Wrap calls in mesos scheduler in future to prevent weird errors. Improve handling of fenzo

Conflicts:
	scheduler/src/cook/mesos/scheduler.clj

Signed-off-by: Wil <41c73e1c187a64e501bf473cfef5419e8eb5a640@twosigma.com>

--- scheduler/src/cook/mesos/scheduler.clj
+++ scheduler/src/cook/mesos/scheduler.clj
@@ -200,94 +200,92 @@
 (defn handle-status-update
   "Takes a status update from mesos."
   [conn driver ^TaskScheduler fenzo status]
-  (future
-    (log/info "Mesos status is: " status)
-    (timers/time!
-      handle-status-update-duration
-      (try (let [db (db conn)
-                 {:keys [task-id reason] task-state :state} status
-                 [job instance prior-instance-status] (first (q '[:find ?j ?i ?status
-                                                                  :in $ ?task-id
-                                                                  :where
-                                                                  [?i :instance/task-id ?task-id]
-                                                                  [?i :instance/status ?s]
-                                                                  [?s :db/ident ?status]
-                                                                  [?j :job/instance ?i]]
-                                                                db task-id))
-                 job-ent (d/entity db job)
-                 instance-ent (d/entity db instance)
-                 retries-so-far (count (:job/instance job-ent))
-                 previous-reason (reason/instance-entity->reason-entity db instance-ent)
-                 instance-status (condp contains? task-state
-                                   #{:task-staging} :instance.status/unknown
-                                   #{:task-starting
-                                     :task-running} :instance.status/running
-                                   #{:task-finished} :instance.status/success
-                                   #{:task-failed
-                                     :task-killed
-                                     :task-lost
-                                     :task-error} :instance.status/failed)
-                 prior-job-state (:job/state (d/entity db job))
-                 progress (try
-                              (when (:data status)
-                                  (:percent (read-string (String. (:data status)))))
+  (log/info "Mesos status is: " status)
+  (timers/time!
+    handle-status-update-duration
+    (try (let [db (db conn)
+               {:keys [task-id reason] task-state :state} status
+               [job instance prior-instance-status] (first (q '[:find ?j ?i ?status
+                                                                :in $ ?task-id
+                                                                :where
+                                                                [?i :instance/task-id ?task-id]
+                                                                [?i :instance/status ?s]
+                                                                [?s :db/ident ?status]
+                                                                [?j :job/instance ?i]]
+                                                              db task-id))
+               job-ent (d/entity db job)
+               instance-ent (d/entity db instance)
+               retries-so-far (count (:job/instance job-ent))
+               previous-reason (reason/instance-entity->reason-entity db instance-ent)
+               instance-status (condp contains? task-state
+                                 #{:task-staging} :instance.status/unknown
+                                 #{:task-starting
+                                   :task-running} :instance.status/running
+                                 #{:task-finished} :instance.status/success
+                                 #{:task-failed
+                                   :task-killed
+                                   :task-lost} :instance.status/failed)
+               prior-job-state (:job/state (d/entity db job))
+               progress (try
+                          (when (:data status)
+                            (:percent (read-string (String. (:data status)))))
                           (catch Exception e
-                              (log/debug e "Error parse mesos status data. Is it in the format we expect?")))
-                 instance-ent (d/entity db instance)
-                 instance-runtime (- (.getTime (now)) ; Used for reporting
-                                     (.getTime (:instance/start-time instance-ent)))
-                 job-resources (util/job-ent->resources job-ent)]
-             (when (#{:instance.status/success :instance.status/failed} instance-status)
-               (log/debug "Unassigning task" task-id "from" (:instance/hostname instance-ent))
-               (try
-                 (.. fenzo
-                     (getTaskUnAssigner)
-                     (call task-id (:instance/hostname instance-ent)))
-                 (catch Exception e
-                   (log/error e "Failed to unassign task" task-id "from" (:instance/hostname instance-ent)))))
-             (when (= instance-status :instance.status/success)
-               (handle-throughput-metrics success-throughput-metrics
-                                          job-resources
-                                          instance-runtime)
-               (handle-throughput-metrics complete-throughput-metrics
-                                          job-resources
-                                          instance-runtime))
-             (when (= instance-status :instance.status/failed)
-               (handle-throughput-metrics fail-throughput-metrics
-                                          job-resources
-                                          instance-runtime)
-               (handle-throughput-metrics complete-throughput-metrics
-                                          job-resources
-                                          instance-runtime))
-             ;; This code kills any task that "shouldn't" be running
-             (when (and
-                     (or (nil? instance) ; We could know nothing about the task, meaning a DB error happened and it's a waste to finish
-                         (= prior-job-state :job.state/completed) ; The task is attached to a failed job, possibly due to instances running on multiple hosts
-                         (= prior-instance-status :instance.status/failed)) ; The kill-task message could've been glitched on the network
-                     (contains? #{:task-running
-                                  :task-staging
-                                  :task-starting}
-                                task-state)) ; killing an unknown task causes a TASK_LOST message. Break the cycle! Only kill non-terminal tasks
-               (log/warn "Attempting to kill task" task-id
-                         "as instance" instance "with" prior-job-state "and" prior-instance-status
-                         "should've been put down already")
-               (meters/mark! tasks-killed-in-status-update)
-               (mesos/kill-task driver task-id))
-             (when-not (nil? instance)
-               ;; (println "update:" task-id task-state job instance instance-status prior-job-state)
-               (log/debug "Transacting updated state for instance" instance "to status" instance-status)
-               (transact-with-retries conn
-                                      (concat
-                                        [[:instance/update-state instance instance-status]]
-                                        (when (and (#{:instance.status/failed} instance-status) (not previous-reason) reason)
-                                          [[:db/add instance :instance/reason (reason/mesos-reason->cook-reason-entity-id db reason)]])
-                                        (when (#{:instance.status/success
-                                                 :instance.status/failed} instance-status)
-                                          [[:db/add instance :instance/end-time (now)]])
-                                        (when progress
-                                          [[:db/add instance :instance/progress progress]])))))
-           (catch Exception e
-             (log/error e "Mesos scheduler status update error"))))))
+                            (log/debug e "Error parse mesos status data. Is it in the format we expect?")))
+               instance-ent (d/entity db instance)
+               instance-runtime (- (.getTime (now)) ; Used for reporting
+                                   (.getTime (:instance/start-time instance-ent)))
+               job-resources (util/job-ent->resources job-ent)]
+           (when (#{:instance.status/success :instance.status/failed} instance-status)
+             (log/debug "Unassigning task" task-id "from" (:instance/hostname instance-ent))
+             (try
+               (.. fenzo
+                   (getTaskUnAssigner)
+                   (call task-id (:instance/hostname instance-ent)))
+               (catch Exception e
+                 (log/error e "Failed to unassign task" task-id "from" (:instance/hostname instance-ent)))))
+           (when (= instance-status :instance.status/success)
+             (handle-throughput-metrics success-throughput-metrics
+                                        job-resources
+                                        instance-runtime)
+             (handle-throughput-metrics complete-throughput-metrics
+                                        job-resources
+                                        instance-runtime))
+           (when (= instance-status :instance.status/failed)
+             (handle-throughput-metrics fail-throughput-metrics
+                                        job-resources
+                                        instance-runtime)
+             (handle-throughput-metrics complete-throughput-metrics
+                                        job-resources
+                                        instance-runtime))
+           ;; This code kills any task that "shouldn't" be running
+           (when (and
+                   (or (nil? instance) ; We could know nothing about the task, meaning a DB error happened and it's a waste to finish
+                       (= prior-job-state :job.state/completed) ; The task is attached to a failed job, possibly due to instances running on multiple hosts
+                       (= prior-instance-status :instance.status/failed)) ; The kill-task message could've been glitched on the network
+                   (contains? #{:task-running
+                                :task-staging
+                                :task-starting}
+                              task-state)) ; killing an unknown task causes a TASK_LOST message. Break the cycle! Only kill non-terminal tasks
+             (log/warn "Attempting to kill task" task-id
+                       "as instance" instance "with" prior-job-state "and" prior-instance-status
+                       "should've been put down already")
+             (meters/mark! tasks-killed-in-status-update)
+             (mesos/kill-task driver task-id))
+           (when-not (nil? instance)
+             ;; (println "update:" task-id task-state job instance instance-status prior-job-state)
+             (log/debug "Transacting updated state for instance" instance "to status" instance-status)
+             (transact-with-retries conn
+                                    (concat
+                                      [[:instance/update-state instance instance-status]]
+                                      (when (and (#{:instance.status/failed} instance-status) (not previous-reason) reason)
+                                        [[:db/add instance :instance/reason (reason/mesos-reason->cook-reason-entity-id db reason)]])
+                                      (when (#{:instance.status/success
+                                               :instance.status/failed} instance-status)
+                                        [[:db/add instance :instance/end-time (now)]])
+                                      (when progress
+                                        [[:db/add instance :instance/progress progress]])))))
+      (catch Exception e
+        (log/error e "Mesos scheduler status update error")))))

 (timers/deftimer [cook-mesos scheduler tx-report-queue-processing-duration])
 (meters/defmeter [cook-mesos scheduler tx-report-queue-datoms])
@@ -539,147 +537,152 @@
   [conn driver ^TaskScheduler fenzo fid pending-jobs num-considerable offers-chan offers]
   (log/debug "invoked handle-resource-offers!")
   (let [offer-stash (atom nil)] ;; This is a way to ensure we never lose offers fenzo assigned if an errors occures in the middle of processing
+    ;; TODO: It is possible to have an offer expire by mesos because we recycle it a bunch of times.
+    ;; TODO: We will see a lot of pending job contention as fenzo takes a long time to schedule. This is *probably* fine as long as some iterations are faster than ranking..
+    ;;       Can we *just* update the new job atom?
+    ;; TODO: If there is an exception before offers are sent to fenzo (scheduleOnce) then the offers will be lost. This is fine with offer expiration, but not great.
     (timers/time!
       handle-resource-offer!-duration
       (try
-        (loop [] ;; This loop is for compare-and-set! below
-          (let [scheduler-contents @pending-jobs
-                db (db conn)
-                _ (log/debug "There are" (count scheduler-contents) "pending jobs")
-                considerable (->> scheduler-contents
-                                  (filter (fn [job]
-                                            (util/job-allowed-to-start? db job)))
-                                  (take num-considerable))
-                _ (log/debug "We'll consider scheduling" (count considerable) "of those pending jobs (limited to " num-considerable " due to backdown)")
-                matches (match-offer-to-schedule fenzo considerable offers db fid)
-                _ (reset! offer-stash (doall (for [{:keys [leases]} matches
-                                                   lease leases]
-                                               (:offer lease))))
-                matched-jobs (for [match matches
-                                   ^TaskAssignmentResult task-result (:tasks match)
-                                   :let [task-request (.getRequest task-result)]]
-                               (:job task-request))
-                processed-matches (process-matches-for-backfill scheduler-contents (first considerable) matched-jobs)
-                new-scheduler-contents (remove (fn [{pending-job-uuid :job/uuid}]
-                                                 (or (contains? (:fully-processed processed-matches) pending-job-uuid)
-                                                     (contains? (:upgrade-backfill processed-matches) pending-job-uuid)))
-                                               scheduler-contents)
-                ;; We don't remove backfilled jobs here, because although backfilled
-                ;; jobs have already been scheduled in a sense, the scheduler still can't
-                ;; adjust the status of backfilled tasks.
-                ;; Backfilled tasks can be updgraded to non-backfilled after the jobs
-                ;; prioritized above them are also scheduled.
-                first-considerable-resources (-> considerable first util/job-ent->resources)
-                match-resource-requirements (util/sum-resources-of-jobs matched-jobs)]
-            (reset! front-of-job-queue-mem-atom
-                    (or (:mem first-considerable-resources) 0))
-            (reset! front-of-job-queue-cpus-atom
-                    (or (:cpus first-considerable-resources) 0))
-            (cond
-              ;; Possible inocuous reasons for no matches: no offers, or no pending jobs.
-              ;; Even beyond that, if Fenzo fails to match ANYTHING, "penalizing" it in the form of giving
-              ;; it fewer jobs to look at is unlikely to improve the situation.
-              ;; "Penalization" should only be employed when Fenzo does successfully match,
-              ;; but the matches don't align with Cook's priorities.
-              (empty? matches) true
-              (not (compare-and-set! pending-jobs scheduler-contents new-scheduler-contents))
-              (do
-                (log/debug "Pending job atom contention encountered, recycling offers:" @offer-stash)
-                (async/go
-                  (async/>! offers-chan @offer-stash))
-                (meters/mark! pending-job-atom-contended)
-                (recur))
-              :else
-              (let [task-txns (for [{:keys [tasks leases]} matches
-                                    :let [offers (mapv :offer leases)
-                                          slave-id (:slave-id (first offers))]
-                                    ^TaskAssignmentResult task tasks
-                                    :let [request (.getRequest task)
-                                          task-info (:task-info request)
-                                          job-id (get-in request [:job :db/id])]]
-                                [[:job/allowed-to-start? job-id]
-                                 {:db/id (d/tempid :db.part/user)
-                                  :job/_instance job-id
-                                  :instance/task-id (:task-id task-info)
-                                  :instance/hostname (.getHostname task)
-                                  :instance/start-time (now)
-                                  ;; NB command executor uses the task-id
-                                  ;; as the executor-id
-                                  :instance/executor-id (get-in
-                                                          task-info
-                                                          [:executor :executor-id]
-                                                          (:task-id task-info))
-                                  :instance/backfilled? (contains? (:backfill-jobs processed-matches) (get-in request [:job :job/uuid]))
-                                  :instance/slave-id slave-id
-                                  :instance/ports (.getAssignedPorts task)
-                                  :instance/progress 0
-                                  :instance/status :instance.status/unknown
-                                  :instance/preempted? false}])
-                    upgrade-txns (map (fn [instance-id]
-                                        [:db/add instance-id :instance/backfilled? false])
-                                      (:upgrade-backfill processed-matches))]
-                ;; Note that this transaction can fail if a job was scheduled
-                ;; during a race. If that happens, then other jobs that should
-                ;; be scheduled will not be eligible for rescheduling until
-                ;; the pending-jobs atom is repopulated
-                @(d/transact
-                   conn
-                   (vec (apply concat upgrade-txns task-txns)))
-                (log/info "Launching" (count task-txns) "tasks")
-                (log/info "Upgrading" (count (:upgrade-backfill processed-matches)) "tasks from backfilled to proper")
-                (log/info "Matched tasks" task-txns)
-                ;; This launch-tasks MUST happen after the above transaction in
-                ;; order to allow a transaction failure (due to failed preconditions)
-                ;; to block the launch
-                (meters/mark! scheduler-offer-matched
-                              (->> matches
-                                   (mapcat (comp :id :offer :leases))
-                                   (distinct)
-                                   (count)))
-                (histograms/update! number-offers-matched
-                                    (->> matches
-                                         (mapcat (comp :id :offer :leases))
-                                         (distinct)
-                                         (count)))
-                (meters/mark! matched-tasks (count task-txns))
-                (meters/mark! matched-tasks-cpus (:cpus match-resource-requirements))
-                (meters/mark! matched-tasks-mem (:mem match-resource-requirements))
-                (doseq [{:keys [tasks leases]} matches
-                        :let [offers (mapv :offer leases)
-                              task-infos (mapv (fn process-results [^TaskAssignmentResult task]
-                                                 (reduce
-                                                   (fn add-ports-to-task-info [task-info [index port]]
-                                                     (log/debug "task-info" task-info [index port])
-                                                     (-> task-info
-                                                         (update-in [:resources :ports]
-                                                                    (fnil conj [])
-                                                                    {:begin port :end port})
-                                                         (assoc-in [:command :environment (str "PORT" index)]
-                                                                   (str port))))
-                                                   (:task-info (.getRequest task))
-                                                   (map-indexed (fn [index port] [index port])
-                                                                (.getAssignedPorts task))))
-                                               tasks)
-                              slave-id (:slave-id (first offers))]]
-                  (mesos/launch-tasks
-                    driver
-                    (mapv :id offers)
-                    (mapv #(-> %
-                               (assoc :slave-id slave-id)
-                               (dissoc :num-ports))
-                          task-infos))
-                  (doseq [^TaskAssignmentResult task tasks]
-                    (.. fenzo
-                        (getTaskAssigner)
-                        (call (.getRequest task) (get-in (first leases) [:offer :hostname])))))
-                (:matched-head? processed-matches)))))
-      (catch Throwable t
-        (meters/mark! handle-resource-offer!-errors)
-        (log/error t "Error in match:" (ex-data t))
-        (async/go
-          (async/>! offers-chan @offer-stash))
-        true  ; if an error happened, it doesn't mean we need to penalize Fenzo
-        )))))
+        (let [scheduler-contents @pending-jobs
+              db (db conn)
+              _ (log/debug "There are" (count scheduler-contents) "pending jobs")
+              considerable (->> scheduler-contents
+                                (filter (fn [job]
+                                          (util/job-allowed-to-start? db job)))
+                                (take num-considerable))
+              _ (log/debug "We'll consider scheduling" (count considerable) "of those pending jobs (limited to " num-considerable " due to backdown)")
+              matches (match-offer-to-schedule fenzo considerable offers db fid)
+              offers-scheduled (for [{:keys [leases]} matches
+                                     lease leases]
+                                 (:offer lease))
+              offers-not-scheduled (clojure.set/intersection (set offers) (set offers-scheduled))
+              _ (reset! offer-stash @offer-stash)
+              matched-jobs (for [match matches
+                                 ^TaskAssignmentResult task-result (:tasks match)
+                                 :let [task-request (.getRequest task-result)]]
+                             (:job task-request))
+              processed-matches (process-matches-for-backfill scheduler-contents (first considerable) matched-jobs)
+              new-scheduler-contents (remove (fn [{pending-job-uuid :job/uuid}]
+                                               (or (contains? (:fully-processed processed-matches) pending-job-uuid)
+                                                   (contains? (:upgrade-backfill processed-matches) pending-job-uuid)))
+                                             scheduler-contents)
+              ;; We don't remove backfilled jobs here, because although backfilled
+              ;; jobs have already been scheduled in a sense, the scheduler still can't
+              ;; adjust the status of backfilled tasks.
+              ;; Backfilled tasks can be updgraded to non-backfilled after the jobs
+              ;; prioritized above them are also scheduled.
+              first-considerable-resources (-> considerable first util/job-ent->resources)
+              match-resource-requirements (util/sum-resources-of-jobs matched-jobs)]
+          (reset! front-of-job-queue-mem-atom
+                  (or (:mem first-considerable-resources) 0))
+          (reset! front-of-job-queue-cpus-atom
+                  (or (:cpus first-considerable-resources) 0))
+          (cond
+            ;; Possible inocuous reasons for no matches: no offers, or no pending jobs.
+            ;; Even beyond that, if Fenzo fails to match ANYTHING, "penalizing" it in the form of giving
+            ;; it fewer jobs to look at is unlikely to improve the situation.
+            ;; "Penalization" should only be employed when Fenzo does successfully match,
+            ;; but the matches don't align with Cook's priorities.
+            (empty? matches) true
+            (not (compare-and-set! pending-jobs scheduler-contents new-scheduler-contents))
+            (do
+              (log/warn "Pending job atom contention encountered, recycling offers:" @offer-stash)
+              (async/go
+                (async/>! offers-chan @offer-stash))
+              (meters/mark! pending-job-atom-contended)
+              true)
+            :else
+            (let [task-txns (for [{:keys [tasks leases]} matches
+                                  :let [offers (mapv :offer leases)
+                                        slave-id (:slave-id (first offers))]
+                                  ^TaskAssignmentResult task tasks
+                                  :let [request (.getRequest task)
+                                        task-info (:task-info request)
+                                        job-id (get-in request [:job :db/id])]]
+                              [[:job/allowed-to-start? job-id]
+                               {:db/id (d/tempid :db.part/user)
+                                :job/_instance job-id
+                                :instance/task-id (:task-id task-info)
+                                :instance/hostname (.getHostname task)
+                                :instance/start-time (now)
+                                ;; NB command executor uses the task-id
+                                ;; as the executor-id
+                                :instance/executor-id (get-in
+                                                        task-info
+                                                        [:executor :executor-id]
+                                                        (:task-id task-info))
+                                :instance/backfilled? (contains? (:backfill-jobs processed-matches) (get-in request [:job :job/uuid]))
+                                :instance/slave-id slave-id
+                                :instance/ports (.getAssignedPorts task)
+                                :instance/progress 0
+                                :instance/status :instance.status/unknown
+                                :instance/preempted? false}])
+                  upgrade-txns (map (fn [instance-id]
+                                      [:db/add instance-id :instance/backfilled? false])
+                                    (:upgrade-backfill processed-matches))]
+              ;; Note that this transaction can fail if a job was scheduled
+              ;; during a race. If that happens, then other jobs that should
+              ;; be scheduled will not be eligible for rescheduling until
+              ;; the pending-jobs atom is repopulated
+              @(d/transact
+                 conn
+                 (vec (apply concat upgrade-txns task-txns)))
+              (log/info "Launching" (count task-txns) "tasks")
+              (log/info "Upgrading" (count (:upgrade-backfill processed-matches)) "tasks from backfilled to proper")
+              (log/info "Matched tasks" task-txns)
+              ;; This launch-tasks MUST happen after the above transaction in
+              ;; order to allow a transaction failure (due to failed preconditions)
+              ;; to block the launch
+              (meters/mark! scheduler-offer-matched
+                            (->> matches
+                                 (mapcat (comp :id :offer :leases))
+                                 (distinct)
+                                 (count)))
+              (histograms/update! number-offers-matched
+                                  (->> matches
+                                       (mapcat (comp :id :offer :leases))
+                                       (distinct)
+                                       (count)))
+              (meters/mark! matched-tasks (count task-txns))
+              (meters/mark! matched-tasks-cpus (:cpus match-resource-requirements))
+              (meters/mark! matched-tasks-mem (:mem match-resource-requirements))
+              (doseq [{:keys [tasks leases]} matches
+                      :let [offers (mapv :offer leases)
+                            task-infos (mapv (fn process-results [^TaskAssignmentResult task]
+                                               (reduce
+                                                 (fn add-ports-to-task-info [task-info [index port]]
+                                                   (log/debug "task-info" task-info [index port])
+                                                   (-> task-info
+                                                       (update-in [:resources :ports]
+                                                                  (fnil conj [])
+                                                                  {:begin port :end port})
+                                                       (assoc-in [:command :environment (str "PORT" index)]
+                                                                 (str port))))
+                                                 (:task-info (.getRequest task))
+                                                 (map-indexed (fn [index port] [index port])
+                                                              (.getAssignedPorts task))))
+                                             tasks)
+                            slave-id (:slave-id (first offers))]]
+                (mesos/launch-tasks
+                  driver
+                  (mapv :id offers)
+                  (mapv #(-> %
+                             (assoc :slave-id slave-id)
+                             (dissoc :num-ports))
+                        task-infos))
+                (doseq [^TaskAssignmentResult task tasks]
+                  (.. fenzo
+                      (getTaskAssigner)
+                      (call (.getRequest task) (get-in (first leases) [:offer :hostname])))))
+              (:matched-head? processed-matches))))
+        (catch Throwable t
+          (meters/mark! handle-resource-offer!-errors)
+          (log/error t "Error in match:" (ex-data t))
+          (async/go
+            (async/>! offers-chan @offer-stash))
+          true  ; if an error happened, it doesn't mean we need to penalize Fenzo
+          )))))

 (defn view-incubating-offers
   [^TaskScheduler fenzo]
@@ -697,12 +700,14 @@
 (gauges/defgauge [cook-mesos scheduler fenzo-num-considerable] (fn [] @fenzo-num-considerable-atom))
 (counters/defcounter [cook-mesos scheduler iterations-at-fenzo-floor])
 (meters/defmeter [cook-mesos scheduler fenzo-abandon-and-reset-meter])
+(counters/defcounter [cook-mesos scheduler offer-chan-depth])

 (defn make-offer-handler
   [conn driver-atom fenzo fid-atom pending-jobs-atom
    max-considerable scaleback
    floor-iterations-before-warn floor-iterations-before-reset]
-  (let [offers-chan (async/chan (async/buffer 5))
+  (let [chan-length 100
+        offers-chan (async/chan (async/buffer chan-length))
         resources-atom (atom (view-incubating-offers fenzo))
         timer-chan (chime-ch (periodic/periodic-seq (time/now) (time/seconds 1))
                              {:ch (async/chan (async/sliding-buffer 1))})]
@@ -714,9 +719,16 @@
         (let [next-considerable
               (try
                 (let [offers (async/alt!!
-                               offers-chan ([offers] offers)
+                               offers-chan ([offers]
+                                            (counters/dec! offer-chan-depth)
+                                            offers)
                                timer-chan ([_] [])
                                :priority true)
+                      ;; Try to clear the channel
+                      offers (->> (repeatedly chan-length #(async/poll! offers-chan))
+                                  (filter nil?)
+                                  (apply concat offers))
+                      _ (log/debug "Passing following offers to handle-resource-offers!" offers)
                       matched-head? (handle-resource-offers! conn @driver-atom fenzo @fid-atom pending-jobs-atom num-considerable offers-chan offers)]
                   (when (seq offers)
                     (reset! resources-atom (view-incubating-offers fenzo)))
@@ -1046,6 +1058,7 @@
                         (rank-jobs (db conn) (d/db conn) offensive-job-filter))))))

 (meters/defmeter [cook-mesos scheduler mesos-error])
+(meters/defmeter [cook-mesos scheduler offer-chan-full-error])

 (defn make-fenzo-scheduler
   [driver offer-incubate-time-ms]
@@ -1060,7 +1073,10 @@
                                        id (:id offer)]
                                    (log/debug "Fenzo is declining offer" offer)
                                    (if-let [driver @driver]
-                                     (mesos/decline-offer driver id)
+                                     (try
+                                       (decline-offers driver [id])
+                                       (catch Exception e
+                                         (log/error e "Unable to decline fenzos rejected offers")))
                                      (log/error "Unable to decline offer; no current driver"))))))
       (build)))

@@ -1071,10 +1087,6 @@
         rescinded-offer-id-cache (-> {}
                                      (cache/fifo-cache-factory :threshold 10240)
                                      atom)
-        offer-chan (async/chan 100)
-        offer-ready-chan (async/chan 100)
-        matcher-chan (async/chan) ; Don't want to buffer offers to the matcher chan. Only want when ready
-
         fenzo (make-fenzo-scheduler driver-atom offer-incubate-time-ms)
         [offers-chan resources-atom] (make-offer-handler conn driver-atom fenzo fid pending-jobs-atom fenzo-max-jobs-considered fenzo-scaleback fenzo-floor-iterations-before-warn fenzo-floor-iterations-before-reset)]
     (start-jobs-prioritizer! conn pending-jobs-atom task-constraints)
@@ -1084,6 +1096,8 @@
                   (log/info "Registered with mesos with framework-id " framework-id)
                   (reset! fid framework-id)
                   (set-framework-id framework-id)
+                  ;; Use future because the thread that runs mesos/scheduler doesn't load classes correctly. for reasons.
+                  ;; As Sophie says, you want to future proof your code.
                   (future
                     (try
                       (reconcile-jobs conn)
@@ -1112,7 +1126,7 @@
              (meters/mark! mesos-error)
              (log/error "Got a mesos error!!!!" message))
       (resourceOffers [driver offers]
-                      (log/debug "Got an offer, putting it into the offer channel:" offers)
+                      (log/info "Got offers, putting them into the offer channel:" offers)
                       (doseq [offer offers]
                         (histograms/update!
                          offer-size-cpus
@@ -1121,8 +1135,15 @@
                          offer-size-mem
                          (get-in offer [:resources :mem] 0)))

-                      (when-not (async/offer! offers-chan offers)
-                        (decline-offers driver offers)))
+                      (if (async/offer! offers-chan offers)
+                        (counters/inc! offer-chan-depth)
+                        (do (log/warn "Offer chan is full. Are we not handling offers fast enough?")
+                            (meters/mark! offer-chan-full-error)
+                            (future
+                              (try
+                                (decline-offers driver (map :id offers))
+                                (catch Exception e
+                                  (log/error e "Unable to decline offers!")))))))
       (statusUpdate [driver status]
-                    (handle-status-update conn driver fenzo status)))
+                    (future (handle-status-update conn driver fenzo status))))
      :view-incubating-offers (fn get-resources-atom [] @resources-atom)}))


Add comments to reactive task killer

Add comments to reactive task killer

--- scheduler/src/cook/mesos/scheduler.clj
+++ scheduler/src/cook/mesos/scheduler.clj
@@ -209,10 +209,11 @@
                (handle-throughput-metrics complete-throughput-metrics
                                           job-resources
                                           instance-runtime))
-             (when (or (nil? instance)
+             ;; This code kills any task that "shouldn't" be running
+             (when (or (nil? instance) ; We could know nothing about the task, meaning a DB error happened and it's a waste to finish
                        (not= task-state :task-lost) ; killing an unknown task causes a TASK_LOST message. Break the cycle!
-                       (= prior-job-state :job.state/completed)
-                       (= prior-instance-status :instance.status/failed))
+                       (= prior-job-state :job.state/completed) ; The task is attached to a failed job, possibly due to instances running on multiple hosts
+                       (= prior-instance-status :instance.status/failed)) ; The kill-task message could've been glitched on the network
                (log/warn "Attempting to kill task" task-id
                          "as instance" instance "with" prior-job-state "and" prior-instance-status
                          "should've been put down already")


Docklands Crackdown and Lemuria Codecracker

--- src/node/game/cards.cljs
+++ src/node/game/cards.cljs
@@ -369,6 +369,9 @@
     :trash-effect {:req (req access)
                    :effect (effect (move :runner card :scored) (gain :runner :agenda-point 2))}}

+   "Docklands Crackdown"
+   {:abilities [{:cost [:click 2] :msg "add 1 power counter" :effect (effect (add-prop card :counter 1))}]}
+
    "Domestic Sleepers"
    {:abilities [{:cost [:click 3] :msg "place 1 agenda counter on Domestic Sleepers"
                  :effect #(do (when (zero? (:counter %3)) (gain-agenda-point %1 %2 1))
@@ -780,6 +783,10 @@

    "Legwork"
    {:effect (effect (run :hq) (access-bonus 2))}
+
+   "Lemuria Codecracker"
+   {:abilities [{:cost [:click 1 :credit 1] :req (req (some #{:hq} (:successful-run runner-reg)))
+                 :msg "expose 1 card"}]}

    "Levy AR Lab Access"
    {:effect (effect (shuffle-into-deck :hand :discard) (draw 5)


use authenticated link

use authenticated link

--- src/oc/email/content.clj
+++ src/oc/email/content.clj
@@ -558,8 +558,8 @@
   (let [notification (:notification msg)
         content (:content notification)
         org (:org msg)
-        board (:board notification)
-        board-slug (:slug board)
+        entry-data (:entry-data notification)
+        board-slug (:board-slug entry-data)
         logo-url (:logo-url org)
         logo? (not (s/blank? logo-url))
         org-name (:name org)
@@ -578,7 +578,8 @@
         notification-author (:author notification)
         notification-author-name (:name notification-author)
         notification-author-url (fix-avatar-url (:avatar-url notification-author))
-        uuid (:uuid content)
+        uuid (:entry-id notification)
+        secure-uuid (:secure-uuid notification)
         origin-url config/web-url
         token-claims {:org-id (:org-id notification)
                       :secure-uuid secure-uuid
@@ -603,7 +604,7 @@
       (spacer 64)
       (h1 intro)
       (spacer 24)
-      (post-block (:entry-data notification) entry-url)
+      (post-block entry-data entry-url)
       (spacer 24)
       (spacer 24 "note-paragraph top-note-paragraph" "note-paragraph top-note-paragraph")
       (note-author notification-author-name notification-author-url)


Fixed a bug in the opponent and spectrally-spaced execution plans.


Add restart button

Add restart button

--- src/nightmod/screens.clj
+++ src/nightmod/screens.clj
@@ -87,6 +87,7 @@
     (update! screen :camera (orthographic) :renderer (stage))
     (let [ui-skin (skin "uiskin.json")]
       [(-> [(text-button "Home" ui-skin :set-name "home")
+            (text-button "Restart" ui-skin :set-name "restart")
             (text-button "Files" ui-skin :set-name "files")]
            (vertical :pack)
            (assoc :id :menu :x 5))
@@ -114,6 +115,7 @@
     (case (text-button! (:actor screen) :get-name)
       "home" (do (u/toggle-glass! false)
                (set-screen! nightmod main-screen))
+      "restart" (reset! u/project-dir @u/project-dir)
       "files" (u/toggle-glass!)
       nil)
     nil))


Keep the channel alive in the face of failures

--- src/clj/mdr2/repair/async.clj
+++ src/clj/mdr2/repair/async.clj
@@ -4,13 +4,17 @@
    [mdr2.repair.core :as archive]
    [mdr2.queues :as queues]
    [mount.core :refer [defstate]]
-   [mdr2.repair.core :as repair]))
+   [mdr2.repair.core :as repair]
+   [failjure.core :as fail]
+   [clojure.tools.logging :as log]))

 (defstate repair-consumer
   :start (go-loop []
            (when-let [production (<! queues/repair)]
-                   (repair/repair production)
-                   (recur)))
+             (let [p (repair/repair production)]
+               (when (fail/failed? p)
+                 (log/errorf "Failed to repair %s" production)))
+             (recur)))

   :stop (when repair-consumer
           (close! repair-consumer)))


Make null character appear as space for help

--- src/cljc/robinson/render.cljc
+++ src/cljc/robinson/render.cljc
@@ -1905,6 +1905,7 @@
 (defn cp437->unicode
   [c]
   (case (int c)
+      0 \
       3 \u2665
     179 \u2502
     219 \u2588


Remove unused variable.

Remove unused variable.

--- src/clojure/clj_chess/uci.clj
+++ src/clojure/clj_chess/uci.clj
@@ -5,8 +5,6 @@
             [me.raynes.conch.low-level :as sh]))

 (alter-var-root #'*out* (constantly *out*))
-
-(def sf-path "/Users/tord/Documents/Programs/C/Stockfish/dev/Stockfish/src/stockfish")

 (defn- option-as-map [& option-components]
   (let [map (into {} option-components)]


Remove configure

Remove configure

--- src/clojure/nightcode/git.clj
+++ src/clojure/nightcode/git.clj
@@ -366,7 +366,6 @@
            (do-with-dialog! pull! repo (utils/get-string :pulling-changes)))
    :push (fn [& _]
            (do-with-dialog! push! repo (utils/get-string :pushing-changes)))
-   :configure (fn [& _])
    :close editors/close-selected-editor!})

 (defn create-widgets
@@ -377,9 +376,6 @@
    :push (ui/button :id :push
                     :text (utils/get-string :push)
                     :listen [:action (:push actions)])
-   :configure (ui/button :id :configure
-                         :text (utils/get-string :configure)
-                         :listen [:action (:configure actions)])
    :close (ui/button :id :close
                      :text "X"
                      :listen [:action (:close actions)])})


Clear assets before restarting game

Clear assets before restarting game

--- src/clojure/nightmod/game.clj
+++ src/clojure/nightmod/game.clj
@@ -1,6 +1,7 @@
 (ns nightmod.game
   (:require [clojail.core :as jail]
             [clojure.java.io :as io]
+            [nightmod.manager :as manager]
             [nightmod.screens :as screens]
             [nightmod.utils :as u]
             [play-clj.core :refer :all]))
@@ -13,9 +14,14 @@

     (set-game-screen! main-screen text-screen)"
   [& game-screens]
-  (->> (conj (vec game-screens) screens/overlay-screen)
-       (apply set-screen! screens/nightmod)
-       on-gl))
+  (on-gl ; clear existing assets so they are reloaded
+         (doseq [gs game-screens]
+           (-> gs :screen (swap! #(dissoc % :renderer :layers))))
+         (asset-manager! manager/manager :clear)
+         ; set the supplied screen(s) with the overlay screen added at the end
+         (apply set-screen!
+                screens/nightmod
+                (conj (vec game-screens) screens/overlay-screen))))

 (defn restart-game!
   "Causes the core.clj file to be run again."


fix Wrong method implementation of min in sql.clj

fix Wrong method implementation of min in sql.clj

--- src/clojure/sparkling/sql.clj
+++ src/clojure/sparkling/sql.clj
@@ -91,7 +91,7 @@
 (defn sum
   "grouped sum"
   [cols grouped-data]
-  (.min grouped-data (into-array cols)))
+  (.sum grouped-data (into-array cols)))

 (defn agg
   "dataset agg"


Rename chain -> channel

Rename chain -> channel

--- src/clojure_fabric/client.clj
+++ src/clojure_fabric/client.clj
@@ -13,12 +13,33 @@
 ;; Each Client instance can maintain several chains representing channels and the associated
 ;; sub-ledgers.

+;;;
+;;; Note: chain == channel
+;;;
+
 (ns clojure-fabric.client
-  (:require [clojure-fabric.user :as user]))
+  (:require [clojure-fabric.user :as user]
+            [clojure-fabric.channel :as channel])
+  (:import clojure_fabric.channel.Channel))

 (defonce ^:private clients (atom {}))

 (defonce ^:dynamic *client* nil)
+
+(defonce system-channel-name "")
+
+(defmacro via-peer-channel [[channel peer] & body]
+  ~@body)
+
+(defmacro via-system-channel-of-peer [[channel peer] & body]
+  `(via-peer-channel [~channel ~peer]
+                     )
+
+  (let [channel# (or ~channel channel/*channel*)]
+    (assert (instance? Channel channel#) (str "Channel is not valid! - " channel#))
+    ;; FIXME: add peer checking
+    (binding [channel/*channel* (assoc channel# :name system-channel-name)]
+      ~@body)))

 (defmacro with-client-binding
   [client & body]
@@ -27,21 +48,35 @@

 (defrecord Client [channels crypto-suite user-context])

-(defrecord Channel [name peers orderers])
-
 ;;; new_chain
-(defn new-chain!
-  "Initializes a chain instance with the given name. This is really representing the \"Channel\"
-  (as explained above), and this call returns an empty object. To initialize the channel, a list of
-  participating endorsers and orderer peers must be configured first on the returned object.
-  Params:
-        name (str): The name of the chain, recommend using namespaces to avoid collision
-  Returns:
-        (Chain instance): The uninitialized chain instance."
+;;;
+;; "Initializes a chain instance with the given name. This is really representing the \"Channel\"
+;;   (as explained above), and this call returns an empty object. To initialize the channel, a list of
+;;   participating endorsers and orderer peers must be configured first on the returned object.
+;;   Params:
+;;         name (str): The name of the chain, recommend using namespaces to avoid collision
+;;   Returns:
+;;         (Chain instance): The uninitialized chain instance."
+(defn add-channel!
+  ([channel]
+   (add-channel! *client* channel))
+  ([client channel]
+   (let [found (get @(:channels client) name)]
+     (if (= found channel)
+       ;; Implementation Note
+       ;;       The spec doesn't say anything about existing channel.
+       ;;       Return if there is an existing channel with the name.
+       found
+       ;; replace or add
+       (swap! (:channels client) assoc name channel)))))
+
+#_
+(defn new-channel!
+
   ([name]
-   (new-chain! *client* name))
+   (new-channel! *client* name))
   ([client name]
-   (new-chain! *client* name {}))
+   (new-channel! *client* name {}))
   ([client name opts]
    (let [channels (:channels client)]
      (if-let [found (get @channels name)]
@@ -51,11 +86,9 @@
        found
        (swap! channels assoc name (-> opts (assoc :name name) (map->Channel)))))))

-;; Alias
-(defonce new-channel! new-chain!)
-
 ;;; get_chain
-(defn get-chain
+#_
+(defn get-channel
   "Get a chain instance from the state storage. This allows existing chain instances to be saved
   for retrieval later and to be shared among instances of the application. Note that it's the
   application/SDK’s responsibility to record the chain information. If an application is not able to
@@ -68,7 +101,7 @@
   Error:  The state store has not been set
         A chain does not exist under that name"
   ([name]
-   (get-chain *client* name))
+   (get-channel *client* name))
   ([client name]
    (if (nil? client)
      (throw (Exception. "The state store has not been set"))
@@ -77,13 +110,10 @@
        ;; Implementation Note
        ;;       The spec says that Returns is chain Instance or None
        ;;       How it can be None and also throw an exception?
-       (throw (Exception. "A chain does not exist under that name"))))))
-
-;; Alias
-(defonce get-channel get-chain)
+       (throw (Exception. "A channel does not exist under that name"))))))

 ;;; query_chain-info
-(defn query-chain-info
+(defn query-channel-info
   "This is a network call to the designated Peer(s) to discover the chain information.
   The target Peer(s) must be part of the chain in question to be able to return the requested
   information.
@@ -96,19 +126,20 @@
         The target Peer(s) does not know anything about the chain"

   ([name peers]
-   (query-chain-info *client* name peers))
+   (query-channel-info *client* name peers))
   ([client name peers]
-   (let [chain (get-chain name)]
-     (if (= (get-peers chain) peers)
-       (%query-chain-info chain peers)
-       (throw (Exception. "The target Peer(s) does not know anything about the chain"))))))
-
-(defonce query-channel-info query-chain-info)
-
-;;;
-;;;
+   (let [channel (get-channel name)]
+     (if (= (channel/get-peers channel) peers)
+       ;;(%query-channel-info channel peers)
+       :FIXME
+       (throw (Exception. "The target Peer(s) does not know anything about the channel"))))))
+
+;;;
+;;;
+#_
 (defonce ^:private client-state-store (atom {}))
 ;;; set_state_store
+#_
 (defn set-state-store!
   "The SDK should have a built-in key value store implementation (suggest a file-based
   implementation to allow easy setup during development). But production systems would want
@@ -182,6 +213,7 @@

 ;;; get_user_context
+#_
 (defn get-user-context
  "As explained above, the client instance can have an optional state store. The SDK saves enrolled
   users in the storage which can be accessed by authorized users of the application
@@ -199,6 +231,21 @@
   ([client name]
    ;; TBD
    ))
+
+
+;;;
+;;; Not in spec, but in some SDKs
+;;;
+(defn query-installed-chaincodes
+  ([peer]
+   (query-installed-chaincodes channel/*channel* peer))
+  ([channel peer]
+   (via-system-channel-of-peer [channel peer]
+     ;; FIXME: Check channel has this peer
+     (-> peer
+         :user
+         (get-transaction-context (user-context client))))))
+
 ;;;;;;;;;;;
 #_
 (comment
@@ -206,7 +253,7 @@
    (get @clients k))

  (binding [*client* (find-client 10)]
-   (get-chain "bar"))
+   (get-channel "bar"))

  (defn make-client! [k]
    (swap! clients assoc k (map->Client {:channels (atom {})})))


Bump tk-ws-jetty9 to 0.7.2 for pe-jvm-puppet

This was using 0.5.2 which is pretty old, and also had an incompatible
dependency on schema.

--- configs/pe-jvm-puppet/pe-jvm-puppet.clj
+++ configs/pe-jvm-puppet/pe-jvm-puppet.clj
@@ -2,7 +2,7 @@
   :description "Release artifacts for pe-jvm-puppet"
   :pedantic? :abort
   :dependencies [[puppetlabs/pe-jvm-puppet-extensions "{{{pe-jvm-puppet-version}}}"]
-                 [puppetlabs/trapperkeeper-webserver-jetty9 "0.5.2"]]
+                 [puppetlabs/trapperkeeper-webserver-jetty9 "0.7.2"]]

   :uberjar-name "jvm-puppet-release.jar"


bump lein-{ancient,ring}

bump lein-{ancient,ring}

--- lein_profiles.clj
+++ lein_profiles.clj
@@ -1,6 +1,6 @@
 {:user {:plugins [
-                  [lein-ring "0.8.13"]
-                  [lein-ancient "0.5.5"]
+                  [lein-ring "0.9.1"]
+                  [lein-ancient "0.6.1"]
                   [jonase/eastwood "0.2.1"]
                   [cider/cider-nrepl "0.8.2"]
                   ]}}


Updates to support IE8...

Updates to support IE8...

- Animate hiding of label as well as form.
- References #47.

--- src/app/cljs/one/sample/animation.cljs
+++ src/app/cljs/one/sample/animation.cljs
@@ -77,10 +77,12 @@
   "Move the form out of view and the greeting into view. Run when the
   submit button is clicked and the form has valid input."
   []
-  (start (parallel (bind form {:effect :fade :start 1 :end 0 :time 500})
-                   (bind cloud
-                         {:effect :color :time 500} ; Dummy animation for delay purposes
-                         {:effect :fade-in-and-show :time 600}))))
+  (let [e {:effect :fade :start 1 :end 0 :time 500}]
+   (start (parallel (bind form e)
+                    (bind label e)
+                    (bind cloud
+                          {:effect :color :time 500} ; Dummy animation for delay purposes
+                          {:effect :fade-in-and-show :time 600})))))

 (defn show-form
   "Move the greeting cloud out of view and show the form. Run when the


Fix var ordering in QP interface :sweat:

--- src/metabase/driver/query_processor/interface.clj
+++ src/metabase/driver/query_processor/interface.clj
@@ -178,6 +178,29 @@
 (s/defrecord RelativeDatetime [amount :- s/Int
                                unit   :- DatetimeValueUnit])

+
+(declare RValue)
+
+(def ^:private ExpressionOperator (s/named (s/enum :+ :- :* :/) "Valid expression operator"))
+
+(s/defrecord Expression [operator        :- ExpressionOperator
+                         args            :- [(s/recursive #'RValue)]
+                         expression-name :- (s/maybe s/Str)]
+  clojure.lang.Named
+  (getName [_] expression-name))
+
+(def RValue
+  "Schema for anything that can be an [RValue](https://github.com/metabase/metabase/wiki/Query-Language-'98#rvalues) -
+   a `Field`, `Value`, or `Expression`."
+  (s/named (s/cond-pre AnyValue FieldPlaceholderOrExpressionRef Expression)
+           "RValue"))
+
+(def AnyField
+  "Schema for a `FieldPlaceholder`, `AgRef`, or `Expression`."
+  (s/named (s/cond-pre ExpressionRef Expression FieldPlaceholderOrAgRef)
+           "Valid field, ag field reference, or expression reference."))
+
+
 (def LiteralDatetimeString
   "Schema for an MBQL datetime string literal, in ISO-8601 format."
   (s/constrained s/Str u/date-string? "Valid ISO-8601 datetime string literal"))
@@ -286,28 +309,6 @@
   (s/named {:page  IntGreaterThanZero
             :items IntGreaterThanZero}
            "Valid page clause"))
-
-(declare RValue)
-
-(def ^:private ExpressionOperator (s/named (s/enum :+ :- :* :/) "Valid expression operator"))
-
-(s/defrecord Expression [operator        :- ExpressionOperator
-                         args            :- [(s/recursive #'RValue)]
-                         expression-name :- (s/maybe s/Str)]
-  clojure.lang.Named
-  (getName [_] expression-name))
-
-(def RValue
-  "Schema for anything that can be an [RValue](https://github.com/metabase/metabase/wiki/Query-Language-'98#rvalues) -
-   a `Field`, `Value`, or `Expression`."
-  (s/named (s/cond-pre AnyValue FieldPlaceholderOrExpressionRef Expression)
-           "RValue"))
-
-(def AnyField
-  "Schema for a `FieldPlaceholder`, `AgRef`, or `Expression`."
-  (s/named (s/cond-pre ExpressionRef Expression FieldPlaceholderOrAgRef)
-           "Valid field, ag field reference, or expression reference."))
-

 (def Query
   "Schema for an MBQL query."


makes eastwood happy

makes eastwood happy

--- src/open_company/resources/stakeholder_update.clj
+++ src/open_company/resources/stakeholder_update.clj
@@ -12,8 +12,9 @@

 ;; ----- Utility functions -----

-(defn- slug-for [title]
+(defn- slug-for
   "Create a slug for the stakeholder update from the slugified title and a short UUID."
+  [title]
   (str (slug/slugify title) "-" (subs (str (java.util.UUID/randomUUID)) 0 4)))

 (defun- sections-for


don't expose the full node-tree by default - only the user-space (with root (root-group))

--- src/overtone/sc/node.clj
+++ src/overtone/sc/node.clj
@@ -532,28 +532,29 @@
 (defn node-tree
   "Returns a data representation of the synth node tree starting at
   the root group."
-  []
-  (ensure-connected!)
-  (group-node-tree 0))
+  ([] (node-tree (root-group)))
+  ([root]
+     (ensure-connected!)
+     (group-node-tree (to-synth-id root))))

 (defn node-tree-zipper
   "Returns a zipper representing the tree of the specified node or
   defaults to the current node tree"
-  ([] (node-tree-zipper 0))
+  ([] (node-tree-zipper (root-group)))
   ([root]
      (zip/zipper map? :children #(assoc %1 :children %2) (group-node-tree root))))

 (defn node-tree-seq
   "Returns a lazy seq of a depth-first traversal of the tree of the
   specified node defaulting to the current node tree"
-  ([] (node-tree-zipper 0))
+  ([] (node-tree-zipper (root-group)))
   ([root] (zipper-seq (node-tree-zipper root))))

 (defn node-tree-matching-synth-ids
   "Returns a seq of synth ids in the node tree with specific
   root (defaulting to the entire node tree) that match regexp or
   strign."
-  ([re-or-str] (node-tree-matching-synth-ids re-or-str 0))
+  ([re-or-str] (node-tree-matching-synth-ids re-or-str (root-group)))
   ([re-or-str root]
      (let [matcher-fn (if (string? re-or-str)
                         =


Change the digest subtitle copy.

Change the digest subtitle copy.

--- src/oc/email/content.clj
+++ src/oc/email/content.clj
@@ -48,8 +48,8 @@

 (def digest-weekly-title "Your weekly brief")
 (def digest-daily-title "Your daily brief")
-(def digest-message "Hi %s, here are the new posts to the %s digest.")
-(def digest-message-no-name "Here are the new posts to the %s digest.")
+(def digest-message "Hi %s, here are the new posts from your team.")
+(def digest-message-no-name "Here are the new posts from your team.")
 (def digest-go-to-digest-button "go_to_digest")
 (def digest-read-post-button "read_post")

@@ -289,8 +289,8 @@
         digest-url (s/join "/" [config/web-url (:org-slug digest) "all-posts"])
         first-name (:first-name digest)
         subtitle (if (s/blank? first-name)
-                    (format digest-message-no-name (:org-name digest))
-                    (format digest-message first-name (:org-name digest)))]
+                    digest-message-no-name
+                    (format digest-message first-name))]
     [:td {:class "small-12 large-12 columns" :valign "middle" :align "center"}
       (spacer 40)
       (when logo? (org-logo {:org-name (:org-name digest)


added restricted-ns for :kafka

added restricted-ns for :kafka

--- src/onyx/tasks/kafka.clj
+++ src/onyx/tasks/kafka.clj
@@ -29,7 +29,8 @@
    (s/optional-key :kafka/fetch-size) s/Num
    (s/optional-key :kafka/empty-read-back-off) s/Num
    (s/optional-key :kafka/commit-interval) s/Num
-   (s/optional-key :kafka/wrap-with-metadata?) s/Bool})
+   (s/optional-key :kafka/wrap-with-metadata?) s/Bool
+   (os/restricted-ns :kafak) s/Any})

 (s/defn ^:always-validate consumer
   ([task-name :- s/Keyword opts]
@@ -75,7 +76,8 @@
    :kafka/zookeeper s/Str
    :kafka/serializer-fn os/NamespacedKeyword
    :kafka/request-size s/Num
-   (s/optional-key :kafka/no-seal?) s/Bool})
+   (s/optional-key :kafka/no-seal?) s/Bool
+   (os/restricted-ns :kafak) s/Any})

 (s/defn ^:always-validate producer
   ([task-name :- s/Keyword opts]


Make PM button just focus new thread mentioning the user

Instead of immediately creating a new thread and sending the user a
message, which can be somewhat unexpected

--- src/braid/core/client/core/events.cljs
+++ src/braid/core/client/core/events.cljs
@@ -140,30 +140,16 @@
                                              (assoc msg :failed? true)
                                              msg))))}))

-; @TODO: Stop this from sending any messages, and
-; instead open up a chat without sending anything to the server
 (reg-event-fx
   :new-conversation
   (fn [{state :db} [_ data]]
-    (let [message (schema/make-message
-                    {:user-id (helpers/current-user-id state)
-                     :content ":wave:"
-                     :group-id (data :group-id)
-                     :mentioned-tag-ids (data :mentioned-tag-ids)
-                     :mentioned-user-ids (data :mentioned-user-ids)})]
-      {:websocket-send
-       (list
-         [:braid.server/new-message message]
-         2000
-         (fn [reply]
-           (when (not= :braid/ok reply)
-             (dispatch [:braid.notices/display! [(keyword "failed-to-send" (message :id))
-                                                 "Message failed to send!"
-                                                 :error]])
-             (dispatch [:set-message-failed message]))))
-       :db (-> state
-               (helpers/add-message message)
-               (helpers/maybe-reset-temp-thread (data :thread-id)))})))
+    {:db (assoc-in state
+                   [:temp-threads
+                    (data :group-id)
+                    :mentioned-ids]
+                   (set (data :mentioned-user-ids)))
+     :dispatch [:focus-thread
+                (get-in state [:temp-threads (data :group-id) :id])]}))

 (reg-event-fx
   :new-message


adding path

adding path

--- src/ophion/aggregate.clj
+++ src/ophion/aggregate.clj
@@ -137,7 +137,8 @@

 (defn mark
   [label]
-  (let [field (mark-field label)]
+  (let [field (mark-field label)
+        path (first field)]
     [{:$addFields field}
      {:$project
       {(str path "._history") false


Fixed bad var name

Fixed bad var name

--- src/polismath/server.clj
+++ src/polismath/server.clj
@@ -31,7 +31,7 @@
 (defn full-path [filename] (str tmp-dir filename))

 (def private-app-url-base (or (:export-server-url-base env/env) "http://localhost:3000"))
-(def public-url-base "https://pol.is")
+(def public-app-url-base "https://pol.is")
 (defn private-url
   [& path]
   (apply str private-app-url-base "/" path))


Verify "top players"

Verify "top players"

--- src/clj_simple_chart/ncs/discovery.clj
+++ src/clj_simple_chart/ncs/discovery.clj
@@ -116,13 +116,26 @@
                               (sort)
                               (vec)))

-(def top-11-players (->> parsed
-                         (filter #(some #{(:name %)} reserve-field-names))
-                         (sort-by :fldRecoverableLiquids)
-                         (take-last 10)
-                         (sort-by :year)
-                         (mapv :name)))
-
+(def top-11-players (conj (->> parsed
+                            (filter #(some #{(:name %)} reserve-field-names))
+                            (sort-by :fldRecoverableLiquids)
+                            (take-last 10)
+                            (sort-by :year)
+                            (mapv :name))
+                          "JOHAN CASTBERG"))
+
+(test/is (= top-11-players
+            ["EKOFISK"
+             "STATFJORD"
+             "VALHALL"
+             "GULLFAKS"
+             "SNORRE"
+             "TROLL"
+             "OSEBERG"
+             "ÅSGARD"
+             "HEIDRUN"
+             "JOHAN SVERDRUP"
+             "JOHAN CASTBERG"]))

 (def missing-field-production
   (->> parsed