Finsihed wading through intercom cleanup analysis.

--- worksheets/intercom_cleanup.clj
+++ worksheets/intercom_cleanup.clj
@@ -11,6 +11,7 @@
   (:require [gorilla-plot.core :as plot]
             [polismath.intercom :as ic]
             [polismath.poller :as poll]
+            [polismath.pretty-printers :as pp]
             [polismath.utils :refer :all]
             [environ.core :as env]))
 ;; @@
@@ -20,7 +21,7 @@

 ;; @@
 ; Defining some helpers
-(defn head [data [& n]] (take (or n 10) data))
+(defn head [data & [n]] (take (or n 10) data))
 ;; @@
 ;; =>
 ;;; {"type":"html","content":"#'intercom-cleanup/head","value":"#'intercom-cleanup/head"}
@@ -42,36 +43,160 @@

 ; Now digging down a little more... We'd like to find out what's going on with the various discrepencies

-(def good-id-by-email (get-db-users-by-email db-spec (map #(get % "email") users-w-ids)))
+(def good-id-by-email (ic/get-db-users-by-email db-spec (map #(get % "email") users-w-ids)))
 (def good-id-by-email-ids (set (map :uid good-id-by-email)))
 (def bad-emails-good-ids-users (remove #(good-id-by-email-ids (Integer/parseInt (get % "user_id"))) users-w-ids))
 (def bad-emails-good-ids-ids (set (map #(get % "user_id") bad-emails-good-ids-users)))
 (def bad-emails-good-ids-dbrecs
-  (get-db-users-by-uid
+  (ic/get-db-users-by-uid
     db-spec
     (map #(Integer/parseInt (get % "user_id")) bad-emails-good-ids-users)))

-(map :email bad-emails-good-ids-dbrecs)
-
-
-;; @@
-;; =>
-;;; {"type":"html","content":"#'intercom-cleanup/bad-emails-good-ids-users","value":"#'intercom-cleanup/bad-emails-good-ids-users"}
-;; <=
-
-;; @@
-; This is how we get a specific user out of the intercom data based on a uid. This in particular is Rosie Hayam
-(filter #(= "27306" (get % "user_id")) users)
-
-;; @@
-;; =>
-;;; {"type":"list-like","open":"(","close":")","separator":" ","items":[{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""unsubscribed_from_emails"","value":"\"unsubscribed_from_emails\""},{"type":"html","content":"false","value":"false"}],"value":"[\"unsubscribed_from_emails\" false]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""segments"","value":"\"segments\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""segment.list"","value":"\"segment.list\""}],"value":"[\"type\" \"segment.list\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""segments"","value":"\"segments\""},{"type":"list-like","open":"[","close":"]","separator":" ","items":[{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""segment"","value":"\"segment\""}],"value":"[\"type\" \"segment\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""id"","value":"\"id\""},{"type":"html","content":""53096101db346636b5000037"","value":"\"53096101db346636b5000037\""}],"value":"[\"id\" \"53096101db346636b5000037\"]"}],"value":"{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"}"},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""segment"","value":"\"segment\""}],"value":"[\"type\" \"segment\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""id"","value":"\"id\""},{"type":"html","content":""53096101db346636b5000038"","value":"\"53096101db346636b5000038\""}],"value":"[\"id\" \"53096101db346636b5000038\"]"}],"value":"{\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}"}],"value":"[{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]"}],"value":"[\"segments\" [{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]]"}],"value":"{\"type\" \"segment.list\", \"segments\" [{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]}"}],"value":"[\"segments\" {\"type\" \"segment.list\", \"segments\" [{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""remote_created_at"","value":"\"remote_created_at\""},{"type":"html","content":"1409755549546","value":"1409755549546"}],"value":"[\"remote_created_at\" 1409755549546]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""custom_attributes"","value":"\"custom_attributes\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[],"value":"{}"}],"value":"[\"custom_attributes\" {}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""session_count"","value":"\"session_count\""},{"type":"html","content":"3","value":"3"}],"value":"[\"session_count\" 3]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""created_at"","value":"\"created_at\""},{"type":"html","content":"1409754749","value":"1409754749"}],"value":"[\"created_at\" 1409754749]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""name"","value":"\"name\""},{"type":"html","content":"nil","value":"nil"}],"value":"[\"name\" nil]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""last_request_at"","value":"\"last_request_at\""},{"type":"html","content":"1409768749","value":"1409768749"}],"value":"[\"last_request_at\" 1409768749]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""user_id"","value":"\"user_id\""},{"type":"html","content":""27306"","value":"\"27306\""}],"value":"[\"user_id\" \"27306\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""location_data"","value":"\"location_data\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""postal_code"","value":"\"postal_code\""},{"type":"html","content":""55102"","value":"\"55102\""}],"value":"[\"postal_code\" \"55102\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""latitude"","value":"\"latitude\""},{"type":"html","content":"44.92779999999999","value":"44.92779999999999"}],"value":"[\"latitude\" 44.92779999999999]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""longitude"","value":"\"longitude\""},{"type":"html","content":"-93.1162","value":"-93.1162"}],"value":"[\"longitude\" -93.1162]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""country_name"","value":"\"country_name\""},{"type":"html","content":""United States"","value":"\"United States\""}],"value":"[\"country_name\" \"United States\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""region_name"","value":"\"region_name\""},{"type":"html","content":""Minnesota"","value":"\"Minnesota\""}],"value":"[\"region_name\" \"Minnesota\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""country_code"","value":"\"country_code\""},{"type":"html","content":""USA"","value":"\"USA\""}],"value":"[\"country_code\" \"USA\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""continent_code"","value":"\"continent_code\""},{"type":"html","content":""NA"","value":"\"NA\""}],"value":"[\"continent_code\" \"NA\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""timezone"","value":"\"timezone\""},{"type":"html","content":""America/Chicago"","value":"\"America/Chicago\""}],"value":"[\"timezone\" \"America/Chicago\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""location_data"","value":"\"location_data\""}],"value":"[\"type\" \"location_data\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""city_name"","value":"\"city_name\""},{"type":"html","content":""Saint Paul"","value":"\"Saint Paul\""}],"value":"[\"city_name\" \"Saint Paul\"]"}],"value":"{\"postal_code\" \"55102\", \"latitude\" 44.92779999999999, \"longitude\" -93.1162, \"country_name\" \"United States\", \"region_name\" \"Minnesota\", \"country_code\" \"USA\", \"continent_code\" \"NA\", \"timezone\" \"America/Chicago\", \"type\" \"location_data\", \"city_name\" \"Saint Paul\"}"}],"value":"[\"location_data\" {\"postal_code\" \"55102\", \"latitude\" 44.92779999999999, \"longitude\" -93.1162, \"country_name\" \"United States\", \"region_name\" \"Minnesota\", \"country_code\" \"USA\", \"continent_code\" \"NA\", \"timezone\" \"America/Chicago\", \"type\" \"location_data\", \"city_name\" \"Saint Paul\"}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""updated_at"","value":"\"updated_at\""},{"type":"html","content":"1409768749","value":"1409768749"}],"value":"[\"updated_at\" 1409768749]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""user_agent_data"","value":"\"user_agent_data\""},{"type":"html","content":""Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0"","value":"\"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0\""}],"value":"[\"user_agent_data\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""tags"","value":"\"tags\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""tag.list"","value":"\"tag.list\""}],"value":"[\"type\" \"tag.list\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""tags"","value":"\"tags\""},{"type":"list-like","open":"[","close":"]","separator":" ","items":[],"value":"[]"}],"value":"[\"tags\" []]"}],"value":"{\"type\" \"tag.list\", \"tags\" []}"}],"value":"[\"tags\" {\"type\" \"tag.list\", \"tags\" []}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""avatar"","value":"\"avatar\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""avatar"","value":"\"avatar\""}],"value":"[\"type\" \"avatar\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""image_url"","value":"\"image_url\""},{"type":"html","content":"nil","value":"nil"}],"value":"[\"image_url\" nil]"}],"value":"{\"type\" \"avatar\", \"image_url\" nil}"}],"value":"[\"avatar\" {\"type\" \"avatar\", \"image_url\" nil}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""app_id"","value":"\"app_id\""},{"type":"html","content":""nb5hla8s"","value":"\"nb5hla8s\""}],"value":"[\"app_id\" \"nb5hla8s\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""user"","value":"\"user\""}],"value":"[\"type\" \"user\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""id"","value":"\"id\""},{"type":"html","content":""5407267d7898790e5a001789"","value":"\"5407267d7898790e5a001789\""}],"value":"[\"id\" \"5407267d7898790e5a001789\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""companies"","value":"\"companies\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""company.list"","value":"\"company.list\""}],"value":"[\"type\" \"company.list\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""companies"","value":"\"companies\""},{"type":"list-like","open":"[","close":"]","separator":" ","items":[],"value":"[]"}],"value":"[\"companies\" []]"}],"value":"{\"type\" \"company.list\", \"companies\" []}"}],"value":"[\"companies\" {\"type\" \"company.list\", \"companies\" []}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""social_profiles"","value":"\"social_profiles\""},{"type":"list-like","open":"{","close":"}","separator":", ","items":[{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""type"","value":"\"type\""},{"type":"html","content":""social_profile.list"","value":"\"social_profile.list\""}],"value":"[\"type\" \"social_profile.list\"]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""social_profiles"","value":"\"social_profiles\""},{"type":"list-like","open":"[","close":"]","separator":" ","items":[],"value":"[]"}],"value":"[\"social_profiles\" []]"}],"value":"{\"type\" \"social_profile.list\", \"social_profiles\" []}"}],"value":"[\"social_profiles\" {\"type\" \"social_profile.list\", \"social_profiles\" []}]"},{"type":"list-like","open":"","close":"","separator":" ","items":[{"type":"html","content":""email"","value":"\"email\""},{"type":"html","content":"""","value":"\"\""}],"value":"[\"email\" \"\"]"}],"value":"{\"unsubscribed_from_emails\" false, \"segments\" {\"type\" \"segment.list\", \"segments\" [{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]}, \"remote_created_at\" 1409755549546, \"custom_attributes\" {}, \"session_count\" 3, \"created_at\" 1409754749, \"name\" nil, \"last_request_at\" 1409768749, \"user_id\" \"27306\", \"location_data\" {\"postal_code\" \"55102\", \"latitude\" 44.92779999999999, \"longitude\" -93.1162, \"country_name\" \"United States\", \"region_name\" \"Minnesota\", \"country_code\" \"USA\", \"continent_code\" \"NA\", \"timezone\" \"America/Chicago\", \"type\" \"location_data\", \"city_name\" \"Saint Paul\"}, \"updated_at\" 1409768749, \"user_agent_data\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0\", \"tags\" {\"type\" \"tag.list\", \"tags\" []}, \"avatar\" {\"type\" \"avatar\", \"image_url\" nil}, \"app_id\" \"nb5hla8s\", \"type\" \"user\", \"id\" \"5407267d7898790e5a001789\", \"companies\" {\"type\" \"company.list\", \"companies\" []}, \"social_profiles\" {\"type\" \"social_profile.list\", \"social_profiles\" []}, \"email\" \"\"}"}],"value":"({\"unsubscribed_from_emails\" false, \"segments\" {\"type\" \"segment.list\", \"segments\" [{\"type\" \"segment\", \"id\" \"53096101db346636b5000037\"} {\"type\" \"segment\", \"id\" \"53096101db346636b5000038\"}]}, \"remote_created_at\" 1409755549546, \"custom_attributes\" {}, \"session_count\" 3, \"created_at\" 1409754749, \"name\" nil, \"last_request_at\" 1409768749, \"user_id\" \"27306\", \"location_data\" {\"postal_code\" \"55102\", \"latitude\" 44.92779999999999, \"longitude\" -93.1162, \"country_name\" \"United States\", \"region_name\" \"Minnesota\", \"country_code\" \"USA\", \"continent_code\" \"NA\", \"timezone\" \"America/Chicago\", \"type\" \"location_data\", \"city_name\" \"Saint Paul\"}, \"updated_at\" 1409768749, \"user_agent_data\" \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:33.0) Gecko/20100101 Firefox/33.0\", \"tags\" {\"type\" \"tag.list\", \"tags\" []}, \"avatar\" {\"type\" \"avatar\", \"image_url\" nil}, \"app_id\" \"nb5hla8s\", \"type\" \"user\", \"id\" \"5407267d7898790e5a001789\", \"companies\" {\"type\" \"company.list\", \"companies\" []}, \"social_profiles\" {\"type\" \"social_profile.list\", \"social_profiles\" []}, \"email\" \"\"})"}
-;; <=
-
-;; @@
-
-;; @@
-
-;; @@
-
-;; @@
+(println (map :email bad-emails-good-ids-dbrecs))
+(println (map #(get % "email") bad-emails-good-ids-users))
+
+
+;; @@
+;; ->
+;;; (light24bulbs@gmail.com m@bjorkegren.com colinmegill@gmail.com timo.erkkila@gmail.com metasoarous@gmail.com sameubank@gmail.com andrew.j.smith@outlook.com mike@pol.is rosas.ben@gmail.com nina@elanprojects.com.au adrian@principalcreative.com.au rosiehoyem@gmail.com oasidjfoiasdjfio@aosidjfiosjdf nil nil nil nil)
+;;; (                )
+;;;
+;; <-
+;; =>
+;;; {"type":"html","content":"nil","value":"nil"}
+;; <=
+
+;; **
+;;;
+;; **
+
+;; **
+;;; There are a number of ways we could hack this up. It seems like we have those ic records for which there are `user_id`s, and those for whom there are not. For the former, updating the emails and times based on the records fetched by the `user_id -> uid` mapping, should be good, as long as there aren't any such mappings that are bad. On the other hand, for the set fetched by email, they should have the correct `user_id` values, so doing the matching by email should be fine. It's really then just the last set, which don't match by ID or email that we have to worry about.
+;;;
+;;; Let's see what those look like as well.
+;;;
+;;; ## Those matching by id have either no email or the correct email?
+;; **
+
+;; @@
+; This is for figuring out if all the email matching for the good IDs work
+
+(let [ic-users users-w-ids
+      db-users (ic/get-db-users-by-uid
+                 db-spec
+                 (map #(Integer/parseInt (get % "user_id")) ic-users))
+      get-dbuser-email-from-icuser-by-id
+        (fn [u]
+          (->> db-users
+               (filter #(= (:uid %) (Integer/parseInt (get u "user_id"))))
+               (first)
+               (:email)))]
+  (doseq [u ic-users]
+    (let [ic-email (get u "email")
+          db-email (get-dbuser-email-from-icuser-by-id u)]
+      (when-not
+        (or (empty? ic-email)
+            (= ic-email db-email))
+        (println [ic-email db-email])))))
+
+;; @@
+;; =>
+;;; {"type":"html","content":"nil","value":"nil"}
+;; <=
+
+;; **
+;;; OK! That seems to settle it! For every ic user for which we have an id that maps to a user in the db, either we don't have the email for the ic record, or the email matches. That means we should be able to safely just update these by passing the email and id together in the update hash, and it will take care of the emails.
+;;;
+;;; ## Now for the ones without ID or email matches:
+;; **
+
+;; @@
+; Now we figure out what's going on with the ones without ID or email matches
+(let [ic-users users-wo-ids
+      db-users (ic/get-db-users-by-email
+                 db-spec
+                 (map #(get % "email") ic-users))
+      db-users-emails (set (map :email db-users))
+      ic-users-bademail (filter
+                 #(not (db-users-emails (get % "email")))
+                 ic-users)]
+  (println "Count difference of" (count ic-users)
+           "in intercom and" (count db-users) "in db is"
+           (- (count ic-users) (count db-users)))
+  (doseq [u ic-users-bademail]
+    (println
+      (ic/gets u ["id" "name" "email"]))))
+;; @@
+;; ->
+;;; Count difference of 87 in intercom and 83 in db is 4
+;;; [53518179fd3643755a00a286 Brian Hayashi brian.hayashi@realtimevegas.co]
+;;; [5352186115e25868ca030190 Sheikh Shuvo sheikh@startupweekend.org]
+;;; [53ce51450083083acd0003d2 Matti Nelimarkka matti.nelimarkka@hiit.fi]
+;;; [53ff62bb7f4310150100083d Julien Brinas julien@castle.ventures]
+;;;
+;; <-
+;; =>
+;;; {"type":"html","content":"nil","value":"nil"}
+;; <=
+
+;; **
+;;;
+;;; Ah... I think these may be more bad email uniformity issues. I find this:
+;;;
+;;;
+;;;     polisapp::BLUE=> select uid, hname, username, email, is_owner, created, plan from users where email like '%clark%';
+;;;       uid  |     hname      | username |           email           | is_owner |    created    | plan
+;;;     -------+----------------+----------+---------------------------+----------+---------------+------
+;;;       3021 | Arianna Clark  |          | arianna.clark@example.com | f        | 1383284862972 |    0
+;;;      26566 | Don            |          | Donclark@gmail.com        | t        | 1397801313466 |    0
+;;;      26630 | Andrew Clarke  |          | aclarke@samuelfrench.com  | t        | 1398110554656 |    0
+;;;      50862 | Clark Mitchell |          | clarkcsmitchell@gmail.com | t        | 1409111052941 |    0
+;;;      84497 | Quinn Rohlf    |          | qrohlf@lclark.edu         | t        | 1409672505722 |    0
+;;;     (5 rows)
+;;;
+;;; I'm guessing the others are going to have similar issues. Time to normalize...
+;;;
+;;; Interesting...
+;;;
+;;;     polisapp::BLUE=> select uid, hname, username, email, is_owner, created, plan from users where hname = 'Brian Hayashi';
+;;;       uid  |     hname     | username |         email          | is_owner |    created    | plan
+;;;     -------+---------------+----------+------------------------+----------+---------------+------
+;;;      26586 | Brian Hayashi |          | brian@realtimevegas.co | t        | 1397845450433 |    0
+;;;     (1 row)
+;;;
+;;; Looks like maybe we need to catch email updates? brian vs brian.hayashi works. Will have to manually take care of this one through our intercom update function.
+;;;
+;;; Also this one:
+;;;
+;;;     polisapp::BLUE=> select uid, hname, username, email, is_owner, created, plan from users where hname = 'Sheikh Shuvo';
+;;;       uid  |    hname     | username |    email     | is_owner |    created    | plan
+;;;     -------+--------------+----------+--------------+----------+---------------+------
+;;;      26571 | Sheikh Shuvo |          | sheikh@up.co | t        | 1397823287771 |    0
+;;;     (1 row)
+;;;
+;;; ### Caps fixes
+;;; * Don (donclark)
+;;; * Don/Dave Evins
+;;; * Loki Astari
+;;;
+;;; > Note: I reran the panel above this to get the intercom ids, and now only 4 show up there. There were 7 before.
+;;;
+;;; ### Wrong address period:
+;;; * **Brian Hayashi**: brian@realtimevegas.co -> brian.hayashi@realtimevegas.co
+;;; * **Sheikh Shuvo**: sheikh@startupweekend.org -> sheikh@up.co
+;;; * **Matti**: matti.nelimarkka@hiit.fi -> matti.nelimarkka@gmail.com
+;;; * **Julien Brinas**: julien@castle.ventures -> julien@togetheraway.com
+;;;
+;;; Woah.. Wait, ok. There are two entries in Intercom for Julien. One with the bad address, the other not. So did he put his email address in incorrectly, leaving the user not in the database? Cause I can't find the other email address in the database (`castle.ventures`). Let's see if we have this problem with the others:
+;;;
+;;; Ah... interesting. Those names appear elsewhere in the IC db. So people are entering bad/old email addresses that don't save to the DB, and then correcting with more recent ones? OR something!!!
+;;;
+;;; In any case, it seems like we can ignore these 4, since there are updated records for them. We _should_ probably delete the old ones though.
+;;;
+;;;
+;;;
+;; **
+
+;; @@
+
+;; @@


CMR-5104: Updated REPL setup.

CMR-5104: Updated REPL setup.

--- dev-resources/src/cmr/opendap/dev.clj
+++ dev-resources/src/cmr/opendap/dev.clj
@@ -1,35 +1,39 @@
 (ns cmr.opendap.dev
   "CMR OPeNDAP development namespace."
   (:require
-    [cheshire.core :as json]
-    [clojure.data.xml :as xml]
-    [clojure.java.io :as io]
-    [clojure.pprint :refer [pprint]]
-    [clojure.set :as set]
-    [clojure.string :as string]
-    [clojure.tools.namespace.repl :as repl]
-    [clojusc.system-manager.core :as system-api :refer :all]
-    [clojusc.twig :as logger]
-    [cmr.authz.components.caching :as auth-caching]
-    [cmr.opendap.components.caching :as concept-caching]
-    [cmr.opendap.components.config :as config]
-    [cmr.opendap.components.core]
-    [cmr.opendap.http.request :as request]
-    [cmr.opendap.http.response :as response]
-    [cmr.opendap.ous.concepts.variable :as variable]
-    ; [cmr.opendap.ous.util.geog :as geog]
-    [cmr.opendap.results.errors :as errors]
-    [com.stuartsierra.component :as component]
-    [debugger.core :as debug]
-    [org.httpkit.client :as httpc]
-    [ring.util.codec :as codec]
-    [trifl.java :refer [show-methods]]
-    [xml-in.core :as xml-in])
+   [cheshire.core :as json]
+   [clojure.data.xml :as xml]
+   [clojure.java.io :as io]
+   [clojure.pprint :refer [pprint]]
+   [clojure.set :as set]
+   [clojure.string :as string]
+   [clojure.tools.namespace.repl :as repl]
+   [clojusc.system-manager.core :as system-api :refer :all]
+   [clojusc.twig :as logger]
+   [cmr.authz.components.caching :as auth-caching]
+   [cmr.opendap.components.caching :as concept-caching]
+   [cmr.opendap.components.config :as config]
+   [cmr.opendap.components.core]
+   [cmr.opendap.http.request :as request]
+   [cmr.opendap.http.response :as response]
+   [cmr.opendap.ous.concepts.variable :as variable]
+   ; [cmr.opendap.ous.util.geog :as geog]
+   [cmr.opendap.query.impl.wcs :as wcs]
+   [cmr.opendap.query.impl.cmr :as cmr]
+   [cmr.opendap.results.errors :as errors]
+   [com.stuartsierra.component :as component]
+   [debugger.core :as debug]
+   [org.httpkit.client :as httpc]
+   [ring.util.codec :as codec]
+   [trifl.java :refer [show-methods]]
+   [xml-in.core :as xml-in])
   (:import
-    (java.net URI)
-    (java.nio.file Paths)
-    ; (net.sf.geographiclib Geodesic PolygonArea)
-    ))
+   (cmr.opendap.query.impl.cmr CollectionCmrStyleParams)
+   (cmr.opendap.query.impl.wcs CollectionWcsStyleParams)
+   (java.net URI)
+   (java.nio.file Paths)
+   ; (net.sf.geographiclib Geodesic PolygonArea)
+   ))

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;   Initial Setup & Utility Functions   ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;


CMR-117: Changed test to check for 400 status code then commented it out since the middleware currently throws a 500. This will be addressed as a separate issue.

--- int_test/cmr/metadata_db/concept_save_test.clj
+++ int_test/cmr/metadata_db/concept_save_test.clj
@@ -58,7 +58,10 @@
         {:keys [status]} (save-concept concept-with-bad-revision)]
     (is (= status 409))))

-(deftest mdb-save-concept-with-invalid-json-test
+;;; This test is disabled because the middleware is currently returning a
+;;; 500 status code instead of a 400. This will be addressed as a separate
+;;; issue.
+#_(deftest mdb-save-concept-with-invalid-json-test
   "Fail to save a concept if the json is invalid"
   (let [response (client/post "http://localhost:3000/concepts"
                               {:body "some non-json"
@@ -69,7 +72,7 @@
         status (:status response)
         body (cheshire/parse-string (:body response))
         error-messages (get body "errors")]
-    (is (not= status 201))))
+    (is (= status 400))))

 (deftest mdb-save-concept-with-missing-required-parameter
   "Fail to save a concept if a required parameter is missing"


added descriptions of supported options to sample project.clj

--- sample/project.clj
+++ sample/project.clj
@@ -16,7 +16,7 @@

   ;; Uncomment this line if your project doesn't use Clojure. Also
   ;; don't forget to remove respective dependencies.
-  ;; java-only true
+  ;; :java-only true

   :dependencies [[android/clojure "1.4.0"]
                  [neko/neko "2.0.0-beta3"]]
@@ -27,20 +27,54 @@
                                  ;; key you want to sign APKs with.
                                  ;; :keystore-path "/home/user/.android/private.keystore"
                                  ;; :key-alias "mykeyalias"
+
+                                 ;; You can specify these to avoid
+                                 ;; entering them for each rebuild,
+                                 ;; but generally it's a bad idea.
+                                 ;; :keypass "android"
+                                 ;; :storepass "android"
+
                                  :aot :all}}}

   :android {;; Specify the path to the Android SDK directory either
             ;; here or in your ~/.lein/profiles.clj file.
             ;; :sdk-path "/home/user/path/to/android-sdk/"

+            ;; Specify this if your project is a library.
+            ;; :library false
+
+            ;; Uncomment this if dexer fails with OutOfMemoryException.
+            ;; :force-dex-optimize true
+
+            ;; Options to pass to dx executable, former is for general
+            ;; java-related options and later is for 'dex task'-specific options.
+            ;; :dex-opts ["-JXmx4096M"]
+            ;; :dex-aux-opts ["--num-threads=2"]
+
+            ;; Proguard config for "droid create-obfuscated-dex" task.
+            ;; :proguard-conf "proguard.cfg"
+            ;; :proguard-opts ["-printseeds"]
+
             ;; Uncomment this line to be able to use Google API.
             ;; :use-google-api true

-            ;; Uncomment this if dexer fails with OutOfMemoryException
-            ;; :force-dex-optimize true
-
             ;; Use this property to add project dependencies.
             ;; :project-dependencies [ "/path/to/library/project" ]
+
+            ;; Sequence of external jars to "link" against, e.g. android-support-v4.jar
+            ;; :external-classes-paths ["<sdk-home>/extras/android/support/v4/android-support-v4.jar"]
+
+            ;; sequence of paths where native libraries may be found for packaging
+            ;; :native-libraries-paths ["libs"]
+
+            ;; Target version affects api used for compilation.
             :target-version "10"
-            :aot-exclude-ns ["clojure.parallel"]
+
+            ;; Minimum supported version could be specified as well,
+            ;; its meaning is similar to that in AndroidManifest.xml.
+            ;; :min-version "10"
+
+            ;; Sequence of namespaces that should not be compiled.
+            :aot-exclude-ns ["clojure.parallel" "clojure.core.reducers"]
+
             })


CMR-4167 - Added test for ingest variable update.

--- system-int-test/test/cmr/system_int_test/ingest/variable_ingest_test.clj
+++ system-int-test/test/cmr/system_int_test/ingest/variable_ingest_test.clj
@@ -1,6 +1,7 @@
 (ns cmr.system-int-test.ingest.variable-ingest-test
   "CMR variable ingest integration tests."
   (:require
+   [clojure.edn :as edn]
    [clojure.test :refer :all]
    [cmr.common.util :refer [are3]]
    [cmr.mock-echo.client.echo-util :as e]
@@ -12,39 +13,56 @@
 (use-fixtures :each (ingest-util/reset-fixture))

 (deftest variable-ingest-test
-  (testing "ingest a new variable"
-    (let [;; Groups
-          update-group-id (e/get-or-create-group (s/context) "umm-var-guid1")
-          ;; Tokens
-          update-token (e/login (s/context) "umm-var-user1" [update-group-id])
-          ;; Grants
-          update-grant-id (e/grant-group-admin
-                           (assoc (s/context) :token update-token)
-                           update-group-id
-                           :update)
-          variable-data (variable-util/make-variable)]
-      (is (e/permitted? update-token
-                        update-grant-id
-                        update-group-id
-                        (ingest-util/get-ingest-update-acls update-token)))
+  (let [;; Groups
+        update-group-id (e/get-or-create-group (s/context) "umm-var-guid1")
+        ;; Tokens
+        update-token (e/login (s/context) "umm-var-user1" [update-group-id])
+        ;; Grants
+        update-grant-id (e/grant-group-admin
+                         (assoc (s/context) :token update-token)
+                         update-group-id
+                         :update)
+        variable-data (variable-util/make-variable)]
+    (is (e/permitted? update-token
+                      update-grant-id
+                      update-group-id
+                      (ingest-util/get-ingest-update-acls update-token)))
+    (testing "ingest: create a new variable"
       (let [{:keys [status concept-id revision-id]} (variable-util/create-variable
                                                      update-token variable-data)]
         (is (= 201 status))
         (is (= 1 revision-id))
         (is (mdb/concept-exists-in-mdb? concept-id revision-id))
-        (is (= (:name variable-data)
-               (:name (mdb/get-concept concept-id revision-id))))
-        ;; XXX Once CMR-4172 (metdata-db services support) was added, we tried to
-        ;;     enable the following test; this required the work that we've since
-        ;;     put into ticket CMR-4193, which in turn is probably blocked by an
-        ;;     as-yet unfiled ticket for addressing what seems to be an ACL caching
-        ;;     issue. Once those two tickets are resolved, this test will be
-        ;;     enabled and should then pass.
-        ; (variable-util/assert-variable-saved variable-data
-        ;                                      "umm-var-user1"
-        ;                                      concept-id
-        ;                                      revision-id)
-        ))))
+        (let [fetched (mdb/get-concept concept-id revision-id)
+              metadata (edn/read-string (:metadata fetched))]
+          (is (= (:Name variable-data) (:native-id fetched)))
+          (is (= (:LongName variable-data) (:long-name metadata)))
+          ;; XXX Once CMR-4172 (metdata-db services support) was added, we tried to
+          ;;     enable the following test; this required the work that we've since
+          ;;     put into ticket CMR-4193, which in turn is probably blocked by an
+          ;;     as-yet unfiled ticket for addressing what seems to be an ACL caching
+          ;;     issue. Once those two tickets are resolved, this test will be
+          ;;     enabled and should then pass.
+          ; (variable-util/assert-variable-saved variable-data
+          ;                                      "umm-var-user1"
+          ;                                      concept-id
+          ;                                      revision-id)
+          )))
+    (testing "ingest: update an existing variable"
+      (let [new-long-name "A new long name"
+            {:keys [status concept-id revision-id]} (variable-util/update-variable
+                                                     update-token
+                                                     (:Name variable-data)
+                                                     (assoc variable-data
+                                                            :LongName
+                                                            new-long-name))]
+        (is (= 200 status))
+        (is (= 2 revision-id))
+        (is (mdb/concept-exists-in-mdb? concept-id revision-id))
+        (let [fetched (mdb/get-concept concept-id revision-id)
+              metadata (edn/read-string (:metadata fetched))]
+          (is (= (:name variable-data) (:name fetched)))
+          (is (= new-long-name (:long-name metadata))))))))

 (deftest variable-ingest-permissions-test
   (testing "Ingest variable permissions:"
@@ -121,7 +139,7 @@
                                                              variable-data)
               update-response (variable-util/update-variable
                                update-token
-                               "A-name"
+                               (:Name variable-data)
                                variable-data)]
           (testing "create variable status"
             (is (= 201 (:status create-response))))


Update riemann version

Update riemann version

--- satori/images/riemann/project.clj
+++ satori/images/riemann/project.clj
@@ -3,8 +3,7 @@
   :url "http://example.com/FIXME"
   :license {:name "Eclipse Public License"
             :url "http://www.eclipse.org/legal/epl-v10.html"}
-  :dependencies [[org.clojure/clojure "1.8.0"]
+  :dependencies [[org.clojure/clojure "1.9.0"]
                   [org.clojure/data.json "0.2.6"]
-                  [org.clojure/tools.reader "1.0.0-RC1"]
-                  [com.taoensso/carmine "2.16.0"]
-                  [riemann "0.2.13"]])
+                  [com.taoensso/carmine "2.18.1"]
+                  [riemann "0.3.1"]])


Pass cljs namespace to js interop related fns

--- cljs-server/cljs_cheatsheet_server/core.cljs
+++ cljs-server/cljs_cheatsheet_server/core.cljs
@@ -61,7 +61,7 @@

 (hiccups/defhtml inside-fn-link
   ([symbol-name]
-   (inside-fn-link symbol-name cljs-core-ns))
+   (inside-fn-link symbol-name clojure-core-ns))
   ([symbol-name name-space]
    (let [full-name (str name-space "/" symbol-name)
          ;; add this symbol to the docs list
@@ -340,14 +340,14 @@
           [:td.label-9e0b7 "Create Object"]
           [:td.body-885f4
             (literal "#js {}")
-            (fn-link "js-obj")]]
+            (fn-link "js-obj" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "Create Array"]
           [:td.body-885f4
             (literal "#js []")
-            (fn-link "array")
-            (fn-link "make-array")
-            (fn-link "aclone")]]
+            (fn-link "array" cljs-core-ns)
+            (fn-link "make-array" cljs-core-ns)
+            (fn-link "aclone" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "Get Property"]
           [:td.body-885f4
@@ -359,7 +359,7 @@
         [:tr
           [:td.label-9e0b7 "Set Property"]
           [:td.body-885f4
-            [:div.row-5dec8 "(" (inside-fn-link "set!") " (.-innerHTML el) \"Hi!\")"]
+            [:div.row-5dec8 "(" (inside-fn-link "set!" cljs-core-ns) " (.-innerHTML el) \"Hi!\")"]
             [:div.row-5dec8
               [:div.msg-b3d36
                 "Using " [:a {:href cljs-oops-lib-url} "cljs-oops"] " library:"]
@@ -367,27 +367,27 @@
         [:tr
           [:td.label-9e0b7 "Delete Property"]
           [:td.body-885f4
-            (fn-link "js-delete")]]
+            (fn-link "js-delete" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "Convert Between"]
           [:td.body-885f4
-            (fn-link "clj->js")
-            (fn-link "js->clj")]]
+            (fn-link "clj->js" cljs-core-ns)
+            (fn-link "js->clj" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "Type Tests"]
           [:td.body-885f4
-            (fn-link "array?")
-            (fn-link "fn?")
-            (fn-link "number?")
-            (fn-link "object?")
-            (fn-link "string?")]]
+            (fn-link "array?" cljs-core-ns)
+            (fn-link "fn?" cljs-core-ns)
+            (fn-link "number?" cljs-core-ns)
+            (fn-link "object?" cljs-core-ns)
+            (fn-link "string?" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "Exceptions"]
           [:td.body-885f4
-            (fn-link "try")
-            (fn-link "catch")
-            (fn-link "finally")
-            (fn-link "throw")]]
+            (fn-link "try" cljs-core-ns)
+            (fn-link "catch" cljs-core-ns)
+            (fn-link "finally" cljs-core-ns)
+            (fn-link "throw" cljs-core-ns)]]
         [:tr
           [:td.label-9e0b7 "External Library"]
           [:td.body-885f4


rangemap can now

rangemap can now

- fallback-val
- .entryAt (for select-keys)

--- sample/src/sample/rangemap.clj
+++ sample/src/sample/rangemap.clj
@@ -1,20 +1,20 @@
 (ns sample.rangemap)

-(deftype RangeMap [kind ^clojure.lang.IPersistentMap m leftfn rightfn]
+(deftype RangeMap [kind ^clojure.lang.IPersistentMap m leftfn rightfn fallback-val]

   clojure.lang.MapEquivalence

   clojure.lang.IPersistentMap

   (assoc [this k v]
-    (RangeMap. kind (.assoc m k v) leftfn rightfn))
+    (RangeMap. kind (.assoc m k v) leftfn rightfn fallback-val))

   (assocEx [this k v]
-    (RangeMap. kind (.assocEx m k v) leftfn rightfn))
+    (RangeMap. kind (.assocEx m k v) leftfn rightfn fallback-val))

   (without [this k]
-    (RangeMap. kind (.without m k) leftfn rightfn))
+    (RangeMap. kind (.without m k) leftfn rightfn fallback-val))

   clojure.lang.ILookup
@@ -39,7 +39,7 @@
                   lastki))]
       (if-not (nil? idx)
         (get m idx)
-        not-found)))
+        (or not-found fallback-val))))

   clojure.lang.IFn
@@ -59,6 +59,9 @@
   (containsKey [this k]
     (.containsKey m k))

+  (entryAt [this k]
+    (.entryAt m k))
+

   clojure.lang.IPersistentCollection

@@ -66,10 +69,10 @@
     (.equiv m o))

   (empty [this]
-    (RangeMap. kind (.empty m) leftfn rightfn))
+    (RangeMap. kind (.empty m) leftfn rightfn fallback-val))

   (cons [this o]
-    (RangeMap. kind (.cons m o) leftfn rightfn))
+    (RangeMap. kind (.cons m o) leftfn rightfn fallback-val))

   (count [this]
     (.count m))
@@ -102,7 +105,7 @@
     (.hasheq m)))

-(defn range-map [kind & kvs]
+(defn range-map [kind fallback-val & kvs]
   {:pre [(or (= kind :find-ceiling)
              (= kind :find-floor))]}
   (let [[leftfn rightfn] (case kind
@@ -110,10 +113,4 @@
                                           (fn [lastki ki] ki)]
                            :find-ceiling [(fn [lastki ki] ki)
                                           (fn [lastki ki] lastki)])]
-    (->RangeMap kind (apply sorted-map kvs) leftfn rightfn)))
-
-(defn range-map-ceiling [& kvs]
-  (apply range-map :find-ceiling kvs))
-
-(defn range-map-floor [& kvs]
-  (apply range-map :find-floor kvs))
+    (->RangeMap kind (apply sorted-map kvs) leftfn rightfn fallback-val)))


catch everything ever

catch everything ever

Signed-off-by: Chris Granger <5654cde260ae99bc4d30054171b8bcd902efd19a@gmail.com>

--- lein-light-nrepl/src/lighttable/nrepl/eval.clj
+++ lein-light-nrepl/src/lighttable/nrepl/eval.clj
@@ -80,22 +80,22 @@
        :result (if (:verbatim opts)
                  res
                  (clean-serialize res opts))})
-    (catch Exception e
-      (let [trace (exception/clean-trace e)
-            msg (or (.getMessage e) (str e))]
-        {:meta (meta f)
-         :form f
-         :result msg
-         :stack trace
-         :ex true}))
-    (catch AssertionError e
-      (let [trace (exception/clean-trace e)
-            msg (or (.getMessage e) (str e))]
-        {:meta (meta f)
-         :form f
-         :result msg
-         :stack trace
-         :ex true}))))
+    (catch Throwable e
+      (try
+        (let [trace (exception/clean-trace e)
+              msg (or (.getMessage e) (str e))]
+          {:meta (meta f)
+           :form f
+           :result msg
+           :stack trace
+           :ex true})
+        (catch Throwable e22
+          {:meta (meta f)
+           :form f
+           :result (str "ERROR: " e)
+           :stack (str "ERROR: " e)
+           :ex true}
+          )))))

 (defn require|create-ns [ns]
   (try


CMR-4470: Updated tests.

CMR-4470: Updated tests.

--- metadata-db-app/int_test/cmr/metadata_db/int_test/concepts/search_test.clj
+++ metadata-db-app/int_test/cmr/metadata_db/int_test/concepts/search_test.clj
@@ -1,22 +1,19 @@
 (ns cmr.metadata-db.int-test.concepts.search-test
-  "Contains integration tests for getting concepts. Tests gets with various
-  configurations including checking for proper error handling."
-  (:require [clojure.test :refer :all]
-            [clj-http.client :as client]
-            [cheshire.core :as cheshire]
-            [clojure.data]
-            [cmr.metadata-db.int-test.utility :as util]
-            [cmr.metadata-db.services.messages :as msg]
-            [clj-time.core :as t]
-            [cmr.common.time-keeper :as tk]
-            [cmr.common.util :refer [are3]]))
+  "Contains integration tests for searching concepts."
+  (:require
+   [clj-time.core :as t]
+   [clojure.test :refer :all]
+   [cmr.common.time-keeper :as tk]
+   [cmr.common.util :refer [are3]]
+   [cmr.metadata-db.int-test.utility :as util]
+   [cmr.metadata-db.services.messages :as msg]))

 (use-fixtures :each (util/reset-database-fixture {:provider-id "REG_PROV" :small false}
                                                  {:provider-id "REG_PROV1" :small false}
                                                  {:provider-id "SMAL_PROV1" :small true}
                                                  {:provider-id "SMAL_PROV2" :small true}))

-(defn concepts-for-comparison
+(defn- concepts-for-comparison
   "Removes revision-date, transaction-id, and created-at from concepts so they can be compared."
   [concepts]
   (map #(dissoc % :revision-date :transaction-id :created-at) concepts))
@@ -171,92 +168,92 @@
                                                              :short-name "s7"}})]
     (testing "find-with-parameters"
       (are3 [collections params]
-            (= (set collections)
+        (is (= (set collections)
                (set (-> (util/find-concepts :collection params)
                         :concepts
-                        concepts-for-comparison)))
-            "regular provider - provider-id"
-            [coll1 coll2-2 coll5-2] {:provider-id "REG_PROV" :latest true}
-
-            "small provider with tombstones - provider-id"
-            [coll3 coll6-2] {:provider-id "SMAL_PROV1" :latest true}
-
-            "small provider - provider-id"
-            [coll4-3] {:provider-id "SMAL_PROV2" :latest true}
-
-            "regular provider - provider-id, entry-title"
-            [coll1] {:provider-id "REG_PROV" :entry-title "et1" :latest true}
-
-            "small provider - provider-id, entry-title"
-            [coll3] {:provider-id "SMAL_PROV1" :entry-title "et3" :latest true}
-
-            "regular provider - provider-id, entry-id"
-            [coll2-2] {:provider-id "REG_PROV" :entry-id "entry-2" :latest true}
-
-            "small provider - provider-id, entry-id"
-            [coll4-3] {:provider-id "SMAL_PROV2" :entry-id "entry-1" :latest true}
-
-            "regular provider - short-name, version-id"
-            [coll2-2] {:short-name "s2" :version-id "v2" :latest true}
-
-            "small provider - short-name, version-id"
-            [coll4-3] {:short-name "s4" :version-id "v5" :latest true}
-
-            "small provider - match multiple - version-id"
-            [coll3] {:version-id "v3" :latest true}
-
-            ;; This test verifies that the provider-id is being used with the small_providers
-            ;; table. Otherwise we would get both coll3 and coll4 back (see previous test).
-            "small provider - provider-id, version-id"
-            [coll4-3] {:provider-id "SMAL_PROV2" :version-id "v5" :latest true}
-
-            "mixed providers - entry-title"
-            [coll1 coll4-3] {:entry-title "et1" :latest true}
-
-            "exclude-metadata=true"
-            [(dissoc coll3 :metadata)]
-            {:provider-id "SMAL_PROV1" :entry-id "entry-3" :exclude-metadata "true" :latest true}
-
-            "exclude-metadata=false"
-            [coll3]
-            {:provider-id "SMAL_PROV1" :entry-id "entry-3" :exclude-metadata "false" :latest true}
-
-            "regular provider - concept-id"
-            [coll1] {:concept-id (:concept-id coll1) :latest true}
-
-            "small provider - concept-id"
-            [coll4-3] {:concept-id (:concept-id coll4-1) :latest true}
-
-            "concept-id and version-id - latest"
-            [coll4-3] {:concept-id (:concept-id coll4-1) :version-id "v5" :latest true}
-
-            "find none - bad provider-id"
-            [] {:provider-id "PROV_NONE" :latest true}
-
-            "find none - provider-id, bad version-id"
-            [] {:provider-id "REG_PROV" :version-id "v7" :latest true}
-
-            ;; all revisions
-            "provider-id - all revisions"
-            [coll4-1 coll4-2 coll4-3] {:provider-id "SMAL_PROV2"}
-
-            "entry-title - all revisions"
-            [coll2-1 coll2-2] {:entry-title "et2"}
-
-            "entry-title - all revisions - find latest false"
-            [coll2-1 coll2-2] {:entry-title "et2" :latest false}
-
-            "concept-id - all revisions"
-            [coll4-1 coll4-2 coll4-3] {:concept-id (:concept-id coll4-1)}
-
-            "concept-id and version-id - all revisions"
-            [coll4-1] {:concept-id (:concept-id coll4-1) :version-id "v3"}
-
-            "params containing a vector are ORed"
-            [coll1 coll3 coll4-3] {:entry-title ["et1" "et3"] :latest true}
-
-            "multiple ORed param lists are ANDed together"
-            [coll3] {:entry-title ["et1" "et3"] :short-name ["s3" "s5"] :latest true}))))
+                        concepts-for-comparison))))
+        "regular provider - provider-id"
+        [coll1 coll2-2 coll5-2] {:provider-id "REG_PROV" :latest true}
+
+        "small provider with tombstones - provider-id"
+        [coll3 coll6-2] {:provider-id "SMAL_PROV1" :latest true}
+
+        "small provider - provider-id"
+        [coll4-3] {:provider-id "SMAL_PROV2" :latest true}
+
+        "regular provider - provider-id, entry-title"
+        [coll1] {:provider-id "REG_PROV" :entry-title "et1" :latest true}
+
+        "small provider - provider-id, entry-title"
+        [coll3] {:provider-id "SMAL_PROV1" :entry-title "et3" :latest true}
+
+        "regular provider - provider-id, entry-id"
+        [coll2-2] {:provider-id "REG_PROV" :entry-id "entry-2" :latest true}
+
+        "small provider - provider-id, entry-id"
+        [coll4-3] {:provider-id "SMAL_PROV2" :entry-id "entry-1" :latest true}
+
+        "regular provider - short-name, version-id"
+        [coll2-2] {:short-name "s2" :version-id "v2" :latest true}
+
+        "small provider - short-name, version-id"
+        [coll4-3] {:short-name "s4" :version-id "v5" :latest true}
+
+        "small provider - match multiple - version-id"
+        [coll3] {:version-id "v3" :latest true}
+
+        ;; This test verifies that the provider-id is being used with the small_providers
+        ;; table. Otherwise we would get both coll3 and coll4 back (see previous test).
+        "small provider - provider-id, version-id"
+        [coll4-3] {:provider-id "SMAL_PROV2" :version-id "v5" :latest true}
+
+        "mixed providers - entry-title"
+        [coll1 coll4-3] {:entry-title "et1" :latest true}
+
+        "exclude-metadata=true"
+        [(dissoc coll3 :metadata)]
+        {:provider-id "SMAL_PROV1" :entry-id "entry-3" :exclude-metadata "true" :latest true}
+
+        "exclude-metadata=false"
+        [coll3]
+        {:provider-id "SMAL_PROV1" :entry-id "entry-3" :exclude-metadata "false" :latest true}
+
+        "regular provider - concept-id"
+        [coll1] {:concept-id (:concept-id coll1) :latest true}
+
+        "small provider - concept-id"
+        [coll4-3] {:concept-id (:concept-id coll4-1) :latest true}
+
+        "concept-id and version-id - latest"
+        [coll4-3] {:concept-id (:concept-id coll4-1) :version-id "v5" :latest true}
+
+        "find none - bad provider-id"
+        [] {:provider-id "PROV_NONE" :latest true}
+
+        "find none - provider-id, bad version-id"
+        [] {:provider-id "REG_PROV" :version-id "v7" :latest true}
+
+        ;; all revisions
+        "provider-id - all revisions"
+        [coll4-1 coll4-2 coll4-3] {:provider-id "SMAL_PROV2"}
+
+        "entry-title - all revisions"
+        [coll2-1 coll2-2] {:entry-title "et2"}
+
+        "entry-title - all revisions - find latest false"
+        [coll2-1 coll2-2] {:entry-title "et2" :latest false}
+
+        "concept-id - all revisions"
+        [coll4-1 coll4-2 coll4-3] {:concept-id (:concept-id coll4-1)}
+
+        "concept-id and version-id - all revisions"
+        [coll4-1] {:concept-id (:concept-id coll4-1) :version-id "v3"}
+
+        "params containing a vector are ORed"
+        [coll1 coll3 coll4-3] {:entry-title ["et1" "et3"] :latest true}
+
+        "multiple ORed param lists are ANDed together"
+        [coll3] {:entry-title ["et1" "et3"] :short-name ["s3" "s5"] :latest true}))))

 (deftest get-expired-collections-concept-ids
   (let [time-now (tk/now)
@@ -311,32 +308,32 @@
     (testing "find with parameters"
       (testing "latest revsions"
         (are3 [granules params]
-              (= (set (map util/expected-concept granules))
+          (is (= (set (map util/expected-concept granules))
                  (set (-> (util/find-latest-concepts :granule params)
                           :concepts
-                          concepts-for-comparison)))
-              ;; These are the only valid combinations for granules
-              "provider-id, granule-ur"
-              [gran1] {:provider-id "REG_PROV" :granule-ur "G1-UR"}
-
-              "provider-id, native-id"
-              [gran2] {:provider-id "REG_PROV" :native-id "G2-NAT"}
-
-              "no metadata"
-              [(dissoc gran1 :metadata)] {:provider-id "REG_PROV"
-                                          :granule-ur "G1-UR"
-                                          :exclude-metadata true}))
+                          concepts-for-comparison))))
+          ;; These are the only valid combinations for granules
+          "provider-id, granule-ur"
+          [gran1] {:provider-id "REG_PROV" :granule-ur "G1-UR"}
+
+          "provider-id, native-id"
+          [gran2] {:provider-id "REG_PROV" :native-id "G2-NAT"}
+
+          "no metadata"
+          [(dissoc gran1 :metadata)] {:provider-id "REG_PROV"
+                                      :granule-ur "G1-UR"
+                                      :exclude-metadata true}))

       (testing "all revisions"
         (are3 [rev-count params]
-              (= rev-count
+          (is (= rev-count
                  (count (-> (util/find-concepts :granule params)
-                            :concepts)))
-              "provider-id, granule-ur - two revisions"
-              2 {:provider-id "REG_PROV" :granule-ur "G2-UR"}
-
-              "provider-id, native-id - three revisons"
-              3 {:provider-id "REG_PROV":native-id "G1-NAT"})))))
+                            :concepts))))
+          "provider-id, granule-ur - two revisions"
+          2 {:provider-id "REG_PROV" :granule-ur "G2-UR"}
+
+          "provider-id, native-id - three revisons"
+          3 {:provider-id "REG_PROV":native-id "G1-NAT"})))))

 (deftest find-collections-with-invalid-parameters
@@ -361,15 +358,15 @@
         tag2 (util/create-and-save-tag 2 2)]
     (testing "find latest revsions"
       (are3 [tags params]
-            (= (set tags)
+        (is (= (set tags)
                (set (->> (util/find-latest-concepts :tag params)
                          :concepts
-                         (map #(dissoc % :provider-id :revision-date :transaction-id)))))
-            "with metadata"
-            [tag1 tag2] {}
-
-            "exclude metadata"
-            [(dissoc tag1 :metadata) (dissoc tag2 :metadata)] {:exclude-metadata true}))
+                         (map #(dissoc % :provider-id :revision-date :transaction-id))))))
+        "with metadata"
+        [tag1 tag2] {}
+
+        "exclude metadata"
+        [(dissoc tag1 :metadata) (dissoc tag2 :metadata)] {:exclude-metadata true}))

     (testing "find all revisions"
       (let [num-of-tags (-> (util/find-concepts :tag)
@@ -389,27 +386,27 @@
                                                             :version-id "v1"
                                                             :short-name "s1"}})
         coll2 (save-collection "REG_PROV" 2 {:extra-fields {:entry-id "entry-2"
-                                                              :entry-title "et2"
-                                                              :version-id "v1"
-                                                              :short-name "s2"}})
+                                                            :entry-title "et2"
+                                                            :version-id "v1"
+                                                            :short-name "s2"}})
         associated-tag (util/create-and-save-tag 1)
         tag-association1 (util/create-and-save-tag-association coll1 associated-tag 1 3)
         tag-association2 (util/create-and-save-tag-association coll2 associated-tag 2 2)]
     (testing "find latest revisions"
       (are3 [tag-associations params]
-            (= (set tag-associations)
+        (is (= (set tag-associations)
                (set (->> (util/find-latest-concepts :tag-association params)
                          :concepts
-                         (map #(dissoc % :provider-id :revision-date :transaction-id)))))
-
-            "by associated-concept-id"
-            [tag-association1] {:associated-concept-id "C1200000000-REG_PROV"}
-
-            "with metadata"
-            [tag-association1 tag-association2] {}
-
-            "exclude metadata"
-            [(dissoc tag-association1 :metadata) (dissoc tag-association2 :metadata)] {:exclude-metadata true}))
+                         (map #(dissoc % :provider-id :revision-date :transaction-id))))))
+
+        "by associated-concept-id"
+        [tag-association1] {:associated-concept-id "C1200000000-REG_PROV"}
+
+        "with metadata"
+        [tag-association1 tag-association2] {}
+
+        "exclude metadata"
+        [(dissoc tag-association1 :metadata) (dissoc tag-association2 :metadata)] {:exclude-metadata true}))

     (testing "find all revisions"
       (let [num-of-tag-associations (-> (util/find-concepts :tag-association {})
@@ -424,26 +421,36 @@
            (util/find-concepts :tag-association {:provider-id "REG_PROV"})))))

 (deftest find-services
-  (let [serv1 (util/create-and-save-service "REG_PROV" 1 3)
-        serv2 (util/create-and-save-service "REG_PROV" 2 2)]
-    (testing "find latest revsions"
+  (let [;; create two services with the same native-id on different providers with multiple revisions
+        serv1 (util/create-and-save-service "REG_PROV" 1 3)
+        serv2 (util/create-and-save-service "SMAL_PROV1" 1 2)]
+    (testing "find latest revisions"
       (are3 [servs params]
-        (= (set servs)
-           (set (->> (util/find-latest-concepts :service params)
-                     :concepts
-                     (map #(dissoc % :provider-id :revision-date :transaction-id)))))
+        (is (= (set servs)
+               (set (->> (util/find-latest-concepts :service params)
+                         :concepts
+                         concepts-for-comparison))))
+
+        "with metadata search by provider-id"
+        [serv1] {:provider-id "REG_PROV"}
+
         "with metadata search by concept-id"
         [serv1] {:concept-id (:concept-id serv1)}

         "with metadata search by native-id"
-        [serv2] {:native-id (:native-id serv2)}
+        [serv1 serv2] {:native-id (:native-id serv2)}
+
+        "with metadata search by provider-id and native-id"
+        [serv2] {:provider-id "SMAL_PROV1"
+                 :native-id (:native-id serv2)}

         "exclude metadata search by concept-id"
         [(dissoc serv1 :metadata)] {:concept-id (:concept-id serv1)
                                     :exclude-metadata true}

-        "exclude metadata search by native-id"
-        [(dissoc serv2 :metadata)] {:native-id (:native-id serv2)
+        "exclude metadata search by provider-id and native-id"
+        [(dissoc serv2 :metadata)] {:provider-id "SMAL_PROV1"
+                                    :native-id (:native-id serv2)
                                     :exclude-metadata true}

         "no match by native id"
@@ -470,52 +477,71 @@
     (testing "find with parameters"
       (testing "latest revisions"
         (are3 [groups params]
-              (= (set (map util/expected-concept groups))
+          (is (= (set (map util/expected-concept groups))
                  (set (-> (util/find-latest-concepts :access-group params)
                           :concepts
-                          concepts-for-comparison)))
-              ;; These are the only valid combinations for groups
-              "provider-id, native-id"
-              [group1] {:provider-id "REG_PROV" :native-id "native-id 1"}
-
-              "no metadata"
-              [(dissoc group1 :metadata)] {:provider-id "REG_PROV"
-                                           :exclude-metadata true}))
+                          concepts-for-comparison))))
+          ;; These are the only valid combinations for groups
+          "provider-id, native-id"
+          [group1] {:provider-id "REG_PROV" :native-id "native-id 1"}
+
+          "no metadata"
+          [(dissoc group1 :metadata)] {:provider-id "REG_PROV"
+                                       :exclude-metadata true}))

       (testing "all revisions"
         (are3 [rev-count params]
-              (= rev-count
-                 (count (-> (util/find-concepts :access-group params)
-                            :concepts)))
-              "provider-id, native-id - three revisons"
-              3 {:provider-id "REG_PROV" :native-id "native-id 1"}
-              "everything"
-              5 {})))))
+          (= rev-count
+             (count (-> (util/find-concepts :access-group params)
+                        :concepts)))
+          "provider-id, native-id - three revisons"
+          3 {:provider-id "REG_PROV" :native-id "native-id 1"}
+          "everything"
+          5 {})))))

 (deftest find-variables
-  (let [variable1 (util/create-and-save-variable "REG_PROV" 1 3)
-        variable2 (util/create-and-save-variable "REG_PROV" 2 2)]
+  (let [;; create two variables with the same native-id on different providers with multiple revisions
+         variable1 (util/create-and-save-variable "REG_PROV" 1 3)
+         variable2 (util/create-and-save-variable "SMAL_PROV1" 1 2)]
     (testing "find latest revsions"
       (are3 [variables params]
-        (= (set variables)
-           (set (->> (util/find-latest-concepts :variable params)
-                     :concepts
-                     (map #(dissoc % :provider-id :revision-date :transaction-id :created-at)))))
-
-        "with metadata"
-        [variable1 variable2]
-        {:provider-id "REG_PROV"}
-
-        "find none - bad provider-id"
+        (is (= (set variables)
+               (set (->> (util/find-latest-concepts :variable params)
+                         :concepts
+                         concepts-for-comparison))))
+
+        "with metadata search by provider-id"
+        [variable1] {:provider-id "REG_PROV"}
+
+        "with metadata search by concept-id"
+        [variable1] {:concept-id (:concept-id variable1)}
+
+        "with metadata search by native-id"
+        [variable1 variable2] {:native-id (:native-id variable1)}
+
+        "with metadata search by provider-id and native-id"
+        [variable2] {:provider-id "SMAL_PROV1"
+                     :native-id (:native-id variable1)}
+
+        "exclude metadata search by concept-id"
+        [(dissoc variable1 :metadata)] {:concept-id (:concept-id variable1)
+                                        :exclude-metadata true}
+
+        "exclude metadata search by provider-id and native-id"
+        [(dissoc variable2 :metadata)] {:provider-id "SMAL_PROV1"
+                                        :native-id (:native-id variable1)
+                                        :exclude-metadata true}
+
+        "no match by native id"
+        [] {:native-id "foo"}
+
+        "no match by provider-id"
         []
-        {:provider-id "PROV_NONE"}
-
-        "exclude metadata"
-        [(dissoc variable1 :metadata) (dissoc variable2 :metadata)]
-        {:provider-id "REG_PROV" :exclude-metadata true}))
+        {:provider-id "PROV_NONE"}))
+
     (testing "find all revisions"
       (let [num-of-variables (-> :variable
-                                 (util/find-concepts {:provider-id "REG_PROV"})
+                                 (util/find-concepts {:native-id (:native-id variable1)})
                                  :concepts
                                  count)]
         (is (= 5 num-of-variables))))))
@@ -534,10 +560,10 @@
         var-association2 (util/create-and-save-variable-association coll2 associated-variable 2 2)]
     (testing "find latest revisions"
       (are3 [variable-associations params]
-        (= (set variable-associations)
-           (set (->> (util/find-latest-concepts :variable-association params)
-                     :concepts
-                     (map #(dissoc % :provider-id :revision-date :transaction-id)))))
+        (is (= (set variable-associations)
+               (set (->> (util/find-latest-concepts :variable-association params)
+                         :concepts
+                         (map #(dissoc % :provider-id :revision-date :transaction-id))))))

         "by associated-concept-id"
         [var-association1]


Also clean up appending bytes to groups here.

--- proxy-protocol/src/main/clojure/com/mattunderscore/proxy/protocol/v1/Plaintextipv4deserialiser.clj
+++ proxy-protocol/src/main/clojure/com/mattunderscore/proxy/protocol/v1/Plaintextipv4deserialiser.clj
@@ -61,9 +61,7 @@
     ; No more data
     (if (has-potential-address context)
       ; Create the address
-      (create-address-result-from-context (-> context
-                                              (update-in [:groups] conj (:pending context))
-                                              (assoc :pending [])) (rest seq))
+      (create-address-result-from-context (append-pending-to-groups context) (rest seq))
       ; Ended early
       (NeedsMoreDataResult/INSTANCE))))


my info

my info

--- resources/members.clj
+++ resources/members.clj
@@ -3,4 +3,6 @@
              :clojure-gem :clojure-http-client :mire]}
  {:name "Brendan Ribera" :url "http://threebrothers.org/brendan/"
   :projects [:clj-kdtree]}
- {:name "Eric Hanchrow"}]
+ {:name "Eric Hanchrow"}
+ {:name "Kevin Downey"
+  :projects [:clojurebot :Repl :clojure-dependency-grapher]}]


Use name in list and include id in grid-editor for reference.

--- sonic-jam/src/sonic_jam/core.cljs
+++ sonic-jam/src/sonic_jam/core.cljs
@@ -377,9 +377,9 @@
   (let [menu-item-fn (fn [owner] (apply hash-map
                                         (apply concat (map #(let [id (first %)
                                                                   grid (second %)]
-                                                              (if (and (contains? % "name")
-                                                                       (not (= "" (get % "name"))))
-                                                                [(str "copy of " (get % "name") "(" (get % "id") ")") id]
+                                                              (if (and (contains? grid "name")
+                                                                       (not (= "" (get grid "name"))))
+                                                                [(str "copy of " (get grid "name") " (" id ")") id]
                                                                 [(str "copy of " id) id]))
                                                            (om/observe owner (grids))))))
         commit-fn (fn [{:keys [cursor id set-state-ch value owner]}]
@@ -793,6 +793,7 @@
                                               (dom/td nil (closer "{"))
                                               (dom/td nil " name: ")
                                               (dom/td nil (om/build name-editor inputs))
+                                              (dom/td nil (str " (" id ")"))
                                               (dom/td nil " ")
                                               (dom/td nil " bpc: ")
                                               (dom/td nil (om/build bpc-editor inputs))


example for adding schema to a task

example for adding schema to a task

--- src/leiningen/new/onyx_app/src/onyx_app/tasks/math.clj
+++ src/leiningen/new/onyx_app/src/onyx_app/tasks/math.clj
@@ -1,14 +1,23 @@
-(ns {{app-name}}.tasks.math)
+(ns {{app-name}}.tasks.math
+    (:require [schema.core :as s]))

 (defn inc-in-segment
   "A specialized version of update-in that increments a key in a segment"
   [ks segment]
   (update-in segment ks inc))

-(defn inc-key [task-name ks task-opts]
-  {:task {:task-map (merge {:onyx/name task-name
-                            :onyx/type :function
-                            :onyx/fn ::inc-in-segment
-                            ::inc-key ks
-                            :onyx/params [::inc-key]}
-                           task-opts)}})
+(def IncKeyTask
+  {::inc-key [s/Keyword]})
+
+(s/defn inc-key
+  ([task-name :- s/Keyword task-opts]
+   {:task {:task-map (merge {:onyx/name task-name
+                             :onyx/type :function
+                             :onyx/fn ::inc-in-segment
+                             :onyx/params [::inc-key]}
+                            task-opts)}
+    :schema {:task-map IncKeyTask}})
+  ([task-name :- s/Keyword
+    ks :- [s/Keyword]
+    task-opts]
+   (inc-key task-name (merge {::inc-key ks} task-opts))))


Formatting fixes

Formatting fixes

--- frontend/components/add_projects.cljs
+++ frontend/components/add_projects.cljs
@@ -38,9 +38,10 @@
       [:div.other-stuff
        [:div.orgname
         [:a {:on-click #(raise! owner [:selected-add-projects-org {:login login :type type}])} login]
-         [:small.github-url.pull-right
-                 [:a {:href (str "https://github.com/" login), :target "_blank"}
-                  [:i.fa.fa-github-alt ""]]]]]]]))
+        [:small.github-url.pull-right
+         [:a {:href (str "https://github.com/" login)
+              :target "_blank"}
+          [:i.fa.fa-github-alt ""]]]]]]]))

 (defn organization-listing [data owner]
   (reify


apply relevant template updates

apply relevant template updates

--- frontend/components/org_settings.cljs
+++ frontend/components/org_settings.cljs
@@ -19,25 +19,25 @@
   (reify
     om/IRender
     (render [_]
-      (letfn [(nav-links [sects msgs selected]
-                (map (fn [sect msg]
-                       [:li {:class (when (= sect selected) :active)}
-                        [:a {:href (str "#" sect)} msg]])
-                     sects msgs))]
+      (letfn [(nav-links [templates]
+                (map (fn [{:keys [page text]} msg]
+                       [:li {:class (when (= page subpage) :active)}
+                        [:a {:href (str "#" page)} text]])
+                     templates))]
         (html [:div.span3
                [:ul.nav.nav-list.well
                 [:li.nav-header "Organization settings"]
                 [:li.divider]
                 [:li.nav-header "Overview"]
-                (nav-links ["projects" "users"]
-                           ["Projects" "Users"]
-                           subpage)
+                (nav-links [{:page "projects" :text "Projects"}
+                            {:page "users" :text "Users"}])
                 [:li.nav-header "Plan"]
                 (if (frontend.models.plan/can-edit-plan? plan)
-                  (nav-links ["containers" "organizations" "billing" "cancel"]
-                             ["Add containers" "Organizations" "Billing info" "Cancel"]
-                             subpage)
-                  (nav-links ["plan"] ["Choose plan"] subpage))]])))))
+                  (nav-links [{:page "containers" :text "Add containers"}
+                              {:page "organizations" :text "Organization"}
+                              {:page "billing" :text "Billing info"}
+                              {:page "cancel" :text "Cancel"}])
+                  (nav-links [{:page "plan" :text "Choose plan"}]))]])))))

 (defn non-admin-plan [{:keys [login org-id subpage]} owner opts]
   (reify


ClojureScript should declare it wants JSON from Liberator, who will happily oblige

--- src-cljs/kixi/hecuba/main.cljs
+++ src-cljs/kixi/hecuba/main.cljs
@@ -47,6 +47,6 @@
              [:a.view-all {:href "/projects"} "View all »"]
              [:a {:href "/projects/new"} "Add a new project"]]]
            ]))
-  (GET "/projects/" {:handler handler}))
+  (GET "/projects/" {:handler handler :headers {"Accept" "application/json"}}))

 (set! (.-onload js/window) main)


Fix sensor selection for chart display.

Fix sensor selection for chart display.

--- src-cljs/kixi/hecuba/main.cljs
+++ src-cljs/kixi/hecuba/main.cljs
@@ -98,19 +98,19 @@
                                                       (<! in))]
       (let [[start-date end-date] search
             entity-id             (get ids :property)
-            [type device-id]      (str/split (get ids :sensor) #"-")
+            sensor-id             (get ids :sensor)
+            [type device-id]      (str/split sensor-id #"-")
             url                   (str "/3/entities/" entity-id "/devices/" device-id "/measurements/" type "?startDate=" start-date "&endDate=" end-date)]
-        (prn "measurements url: " url)
+        (om/update! data :sensor sensor-id)
         (when (and (not (empty? start-date))
                    (not (empty? end-date))
                    (not (nil? device-id))
                    (not (nil? entity-id))
                    (not (nil? type)))
-          (GET url {:handler #(om/transact! data [:measurements] (constantly %))
+          (GET url {:handler #(om/update! data :measurements %)
                     :headers {"Accept" "application/json"}
                     :response-format :json
-                    :keywords? true})))
-      )
+                    :keywords? true}))))
     (recur)))

@@ -419,7 +419,7 @@
                                             :selection-key :measurement})
           (chart-ajax (tap-history) (:chart data) {:template "/3/entities/:property/devices/:device/measurements?startDate=:start-date&endDate=:end-date"
                                                    :content-type  "application/json"
-                                         :selection-key :range})
+                                                   :selection-key :range})
           ))
       om/IRender
       (render [_]


restore the queue view

restore the queue view

--- cljs/sledge/core.cljs
+++ cljs/sledge/core.cljs
@@ -370,7 +370,7 @@
                            {:index %2
                             :current? (= (dec (:next-track queue)) %2)
                             }})
-               queue (range 0 999))
+               (:tracks queue) (range 0 999))
           ])))))

 (defn search-entry-view [term owner]


Use helpers in c.t.n.move-test

Use helpers in c.t.n.move-test

--- src/test/clojure/clojure/tools/namespace/move_test.clj
+++ src/test/clojure/clojure/tools/namespace/move_test.clj
@@ -1,62 +1,42 @@
 (ns clojure.tools.namespace.move-test
-  (:use [clojure.java.io :as io]
-        [clojure.test :only (deftest is)]
-        [clojure.tools.namespace.move :only (move-ns)])
+  (:require [clojure.java.io :as io]
+            [clojure.test :refer [deftest is]]
+            [clojure.tools.namespace.move :refer [move-ns]]
+            [clojure.tools.namespace.test-helpers :as help])
   (:import (java.io File)))

-(defn- create-temp-dir
-  "Creates and returns a new temporary directory java.io.File."
-  [name]
-  (let [temp-file (File/createTempFile name nil)]
-    (.delete temp-file)
-    (.mkdirs temp-file)
-    temp-file))
+(defn- create-file-one [dir]
+  (help/create-source dir 'example.one :clj
+                      '[example.two example.three]
+                      '[(defn foo []
+                           (example.a.four/foo))]))

-(def ^:private file-one-contents "
- (ns com.example.one
-   (:require [com.example.two :as two]
-             [com.example.three :as three]))
+(defn- create-file-two [dir]
+  (help/create-source dir 'example.two :clj
+                    '[example.three example.a.four]))

- (defn foo []
-   (com.example.a.four/foo))
-")
+(defn- create-file-three [dir]
+  (help/create-source dir 'example.three :clj
+                    '[example.five]))

-(def ^:private file-two-contents "
- (ns com.example.two
-   (:require [com.example.three :as three]
-             [com.example.a.four :as four]))
-")
-
-(def ^:private file-three-contents "
- (ns com.example.three
-   (:use com.example.five))
-")
-
-(def ^:private file-four-contents "
- (ns com.example.a.four)
- (defn foo [] \"foo\")
-")
+(defn- create-file-four [dir]
+  (help/create-source dir 'example.a.four :clj))

 (deftest t-move-ns
-  (let [temp-dir (create-temp-dir "tools-namespace-t-move-ns")
+  (let [temp-dir (help/create-temp-dir "tools-namespace-t-move-ns")
         src-dir (io/file temp-dir "src")
-        example-dir (io/file temp-dir "src" "com" "example")
-        file-one (io/file example-dir "one.clj")
-        file-two (io/file example-dir "two.clj")
-        file-three (io/file example-dir "three.clj")
-        old-file-four (io/file example-dir "a" "four.clj")
+        example-dir (io/file temp-dir "src" "example")
+        file-one (create-file-one src-dir)
+        file-two (create-file-two src-dir)
+        file-three (create-file-three src-dir)
+        old-file-four (create-file-four src-dir)
         new-file-four (io/file example-dir "b" "four.clj")]
-
-    (.mkdirs (io/file example-dir "a"))
-    (spit file-one file-one-contents)
-    (spit file-two file-two-contents)
-    (spit file-three file-three-contents)
-    (spit old-file-four file-four-contents)
-    (Thread/sleep 1500) ;; ensure file timestamps are different

     (let [file-three-last-modified (.lastModified file-three)]

-      (move-ns 'com.example.a.four 'com.example.b.four src-dir [src-dir])
+      (Thread/sleep 1500) ;; ensure file timestamps are different
+
+      (move-ns 'example.a.four 'example.b.four src-dir [src-dir])

       (is (.exists new-file-four)
           "new file should exist")
@@ -66,7 +46,7 @@
           "old empty directory should not exist")
       (is (= file-three-last-modified (.lastModified file-three))
           "unaffected file should not have been modified")
-      (is (not-any? #(.contains (slurp %) "com.example.a.four")
+      (is (not-any? #(.contains (slurp %) "example.a.four")
                     [file-one file-two file-three new-file-four]))
-      (is (every? #(.contains (slurp %) "com.example.b.four")
+      (is (every? #(.contains (slurp %) "example.b.four")
                   [file-one file-two new-file-four])))))


GAWB-1397: format

GAWB-1397: format

--- src/cljs/main/broadfcui/page/workspace/monitor/workflow_details.cljs
+++ src/cljs/main/broadfcui/page/workspace/monitor/workflow_details.cljs
@@ -77,8 +77,7 @@
              [height (aget (aget (aget (aget (.-childNodes (aget (.-childNodes (aget (.-childNodes (aget (.-childNodes (aget
                         (.-childNodes (.getElementById js/document "chart_div")) 0)) 0)) 0)) 1)) 0) "height") "animVal") "value")]
              (gdom/remove-children "chart_div")
-             (.timingDiagram js/window (:data props) (:workflow-name props) (+ 50 height))))
-         0)
+             (.timingDiagram js/window (:data props) (:workflow-name props) (+ 50 height)))) 0)
        (when (.getElementById js/document "chart_div") (gdom/remove-children "chart_div"))))})

 (defn- backend-logs [data]


Always fallback to default container cost

In the case of a piggiebacking org, the :paid plan template doesn't
contain a :container_cost.  Use default-template-properties when a price
is not found, rather than just when a plan is not found (previously the
plan creation page for a piggiebacking org showed containers as not
having a cost.)

--- src-cljs/frontend/models/plan.cljs
+++ src-cljs/frontend/models/plan.cljs
@@ -131,10 +131,9 @@
             (str (time/in-minutes trial-interval) " minutes")))))

 (defn per-container-cost [plan]
-  (let [template-properties (or (-> plan :paid :template)
-                                (-> plan :enterprise :template)
-                                default-template-properties)]
-    (:container_cost template-properties)))
+  (or (-> plan :paid :template :container_cost)
+      (-> plan :enterprise :template :container_cost)
+      (:container_cost default-template-properties)))

 (defn container-cost [plan containers]
   (let [template-properties (or (-> plan :paid :template)


Prefix length for markov chains is now configurable.

--- Spraff/src/spraff/core.clj
+++ Spraff/src/spraff/core.clj
@@ -21,6 +21,7 @@
 (def corpus-file "corpus.txt")
 (def word-pattern #"\S+")
 (def generated-sentence-length 10)
+(def prefix-length 3)

 (def table (ref {}))
 (def starters (ref []))
@@ -32,19 +33,20 @@
         (if (< i weight)
             word
             (recur (rest c) (- i weight)))))
-(defn pick-next-word [a b t]
-    (when-let [candidates (t [a b])]
+(defn pick-next-word [prefix t]
+    (when-let [candidates (t prefix)]
         (select candidates
             (rand-int (reduce + 0 (vals candidates))))))
-(defn further-stream-of-shite [a b t]
+(defn further-stream-of-shite [prefix t]
     (lazy-seq
-        (when-let [c (pick-next-word a b t)]
-            (cons c (further-stream-of-shite b c t)))))
-(defn stream-of-shite [a b t]
-    (concat [a b] (further-stream-of-shite a b t)))
+        (when-let [c (pick-next-word prefix t)]
+            (cons c (further-stream-of-shite (concat (rest prefix) [c]) t)))))
+(defn stream-of-shite [prefix t]
+    (concat prefix (further-stream-of-shite prefix t)))
 (defn generate-sentence []
-    (let [[a b] (nth @starters (rand-int (count @starters)))]
-        (take generated-sentence-length (stream-of-shite a b @table))))
+    (take generated-sentence-length (stream-of-shite
+        (nth @starters (rand-int (count @starters)))
+        @table)))

 (defn value-or-default [value default]
     (if (nil? value) default value))
@@ -52,23 +54,23 @@
     (assoc m
         c
         (inc (value-or-default (m c) 0))))
-(defn update-triple [a b c t]
+(defn update-transition [prefix c t]
     (assoc t
-        [a b]
+        prefix
         (update-count
-            (value-or-default (t [a b]) {})
+            (value-or-default (t prefix) {})
             c)))
 (defn update-table [table words]
-    (if (>= (count words) 3)
-        (let [[a b c & _] words]
-            (recur (update-triple a b c table) (rest words)))
+    (if (> (count words) prefix-length)
+        (let [[prefix [c & _]] (split-at prefix-length words)]
+            (recur (update-transition prefix c table) (rest words)))
         table))
 (defn update-message! [message]
     (dosync
         (let [words (split-sentence-to-words message)]
             (ref-set table (update-table @table words))
-            (if (>= (count words) 2)
-                (ref-set starters (conj @starters (take 2 words)))))))
+            (if (>= (count words) prefix-length)
+                (ref-set starters (conj @starters (take prefix-length words)))))))
 (defn log-sentence! [message]
     (with-open [out (writer corpus-file :append true)]
         (.write out (str message "\n"))


Update src/active/clojure/validation.cljc

Co-authored-by: kaaninho <f950d10c4916556a5d4bff6a6dc331bacbf26608@active-group.de>

--- src/active/clojure/validation.cljc
+++ src/active/clojure/validation.cljc
@@ -100,7 +100,7 @@
   make-validation-error validation-error?
   [^{:doc "The candidate value that was being validated."}
    candidate validation-error-candidate
-   ^{:doc "A message signifying was kind of error occured.  It should
+   ^{:doc "A message signifying what kind of error occured.  It should
 possible for the user to interpret the message as they please, so
 usually a namespaced keyword representing the error works well."}
    message validation-error-message


Fix set-last-stage

Fix set-last-stage

--- src/antlion_clojure/dynamodb.clj
+++ src/antlion_clojure/dynamodb.clj
@@ -157,7 +157,7 @@
   [{:keys [opts] :as comp} name state]
   (far/put-item opts :antlion-clojure-last-state
                 {:name name}
-                {:state state}))
+                {:state [:put state]}))

 (defn set-lemming-last-state
   [{:keys [opts] :as comp} state]