CMR-3357: Cleaned up the reorganized the tests.

--- system-int-test/test/cmr/system_int_test/ingest/collection_validation_test.clj
+++ system-int-test/test/cmr/system_int_test/ingest/collection_validation_test.clj
@@ -23,6 +23,112 @@
     [cmr.system-int-test.utils.search-util :as search]
     [clojure.java.io :as io]))

+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;;
+;; The following tests are included in this file:
+;; 1. Spatial coordinate reference type must be supplied.
+;; 2. Validation end point tests:
+;;    a)successful case
+;;    b)invalid collection xml fails validation with appropriate message
+;;    (Do we want to test invalid umm-json fails validation here?)
+;; 3. successful validation requests do not get an xml or json response body
+;; 4. failed validations with no accept or content-type header return xml
+;; 5. Collection Keyword validation
+;;    a) keyword validation using validation end points.
+;;    b) Project keyword validation
+;;    c) Platform keyword validation
+;;    d) Instrument keyword validation
+;;    e) Science Keyword validation
+;; 6. collection-umm-spec-validation-test
+;;    a)UMM-C JSON-Schema validation through config settings
+;;    b)UMM-C JSON-Schema validation through Cmr-Validate-Umm-C header
+;;    c)UMM-SPEC validation through Cmr-Validation-Umm-C header
+;;    d)Additional Attribute validation
+;;    e)Nested Path Validation
+;;    f)Spatial validation
+;; 7. duplicate-entry-title-test
+;; 8. revision-id-validations
+;; 9. revision-conflict-tests
+;;10. Collections with nil versions are rejected
+;;11. Conversion from dif9 to echo10 is successful
+;;    (Do we want to move other conversions here or leave it in the unit tests?
+;;     do we want to add the conversion through translate end point also?)
+;;12. umm-spec-validation-warnings for all formats
+;;
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;; helper function section
+;;;;;;;;;;;;;;;;;;;;;;;;;;
+(defn polygon
+  "Creates a single ring polygon with the given ordinates. Points must be in counter clockwise order."
+  [& ords]
+  (poly/polygon [(apply umm-s/ords->ring ords)]))
+
+(defn assert-invalid
+  ([coll-attributes field-path errors]
+   (assert-invalid coll-attributes field-path errors nil))
+  ([coll-attributes field-path errors options]
+   (let [response (d/ingest "PROV1" (dc/collection coll-attributes)
+                            (merge {:allow-failure? true} options))]
+     (is (= {:status 422
+             :errors [{:path field-path
+                       :errors errors}]}
+            (select-keys response [:status :errors]))))))
+
+(defn assert-valid
+  ([coll-attributes]
+   (assert-valid coll-attributes nil))
+  ([coll-attributes options]
+   (let [collection (assoc (dc/collection coll-attributes) :native-id (:native-id coll-attributes))
+         provider-id (get coll-attributes :provider-id "PROV1")
+         response (d/ingest provider-id collection options)]
+     (is (#{{:status 200} {:status 201}} (select-keys response [:status :errors]))))))
+
+(defn assert-invalid-keywords
+  [coll-attributes field-path errors]
+  (assert-invalid coll-attributes field-path errors {:validate-keywords true}))
+
+(defn assert-valid-keywords
+  [coll-attributes]
+  (assert-valid coll-attributes {:validate-keywords true}))
+
+(defn assert-invalid-spatial
+  ([coord-sys shapes field-path errors]
+   (assert-invalid-spatial coord-sys shapes field-path errors nil))
+  ([coord-sys shapes field-path errors options]
+   (let [shapes (map (partial umm-s/set-coordinate-system coord-sys) shapes)]
+     (assert-invalid {:spatial-coverage (dc/spatial {:gsr coord-sys
+                                                     :sr coord-sys
+                                                     :geometries shapes})}
+                     field-path
+                     errors
+                     options))))
+
+(defn assert-valid-spatial
+  [coord-sys shapes]
+  (let [shapes (map (partial umm-s/set-coordinate-system coord-sys) shapes)]
+    (assert-valid {:spatial-coverage (dc/spatial {:gsr coord-sys
+                                                  :sr coord-sys
+                                                  :geometries shapes})})))
+
+(defn assert-conflict
+  [coll-attributes errors]
+  (let [collection (assoc (dc/collection coll-attributes) :native-id (:native-id coll-attributes))
+        response (d/ingest "PROV1" collection {:allow-failure? true})]
+    (is (= {:status 409
+            :errors errors}
+           (select-keys response [:status :errors])))))
+
+(defn- assert-revision-conflict
+  [concept-id format-str response]
+  (is (= {:status 409
+          :errors [(format format-str concept-id)]}
+         response)))
+
+;;;;;;;;;;;;;;;;;;;;
+;; Testing section
+;;;;;;;;;;;;;;;;;;;;
 (use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"}))

 (deftest spatial-with-no-representation
@@ -53,86 +159,25 @@
                    "Line 1 - cvc-complex-type.2.4.b: The content of element 'Collection' is not complete. One of '{ShortName}' is expected."]]
              [status errors])))))

-;; Verify that successful validation requests do not get an xml or json response body
 (deftest successful-validation-with-accept-header-test
-  (are [accept]
-       (let [collection (dc/collection-dif10 {:processing-level-id "Level 1"})
-             concept (dc/collection-concept collection :dif10)
-             response-map (select-keys (ingest/validate-concept concept {:accept-format accept :raw? true})
-                                       [:status :body])]
-         (= {:status 200 :body ""} response-map))
-       :json :xml))
-
-;; Verify that failed validations with no accept or content-type header return xml
+  (testing "successful validation requests do not get an xml or json response body"
+    (are [accept]
+         (let [collection (dc/collection-dif10 {:processing-level-id "Level 1"})
+               concept (dc/collection-concept collection :dif10)
+               response-map (select-keys (ingest/validate-concept concept {:accept-format accept :raw? true})
+                                         [:status :body])]
+           (= {:status 200 :body ""} response-map))
+         :json :xml)))
+
 (deftest failed-validation-without-headers-returns-xml
-  (let [concept (assoc (dc/collection-concept {}) :metadata "<Collection>invalid xml</Collection>")
-        {:keys [status body]} (ingest/validate-concept concept {:raw? true})]
-    (is (= [400 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><errors><error>Line 1 - cvc-complex-type.2.3: Element 'Collection' cannot have character [children], because the type's content type is element-only.</error><error>Line 1 - cvc-complex-type.2.4.b: The content of element 'Collection' is not complete. One of '{ShortName}' is expected.</error></errors>"]
-           [status body]))))
-
-(defn polygon
-  "Creates a single ring polygon with the given ordinates. Points must be in counter clockwise order."
-  [& ords]
-  (poly/polygon [(apply umm-s/ords->ring ords)]))
-
-(defn assert-invalid
-  ([coll-attributes field-path errors]
-   (assert-invalid coll-attributes field-path errors nil))
-  ([coll-attributes field-path errors options]
-   (let [response (d/ingest "PROV1" (dc/collection coll-attributes)
-                            (merge {:allow-failure? true} options))]
-     (is (= {:status 422
-             :errors [{:path field-path
-                       :errors errors}]}
-            (select-keys response [:status :errors]))))))
-
-(defn assert-invalid-keywords
-  [coll-attributes field-path errors]
-  (assert-invalid coll-attributes field-path errors {:validate-keywords true}))
-
-(defn assert-valid
-  ([coll-attributes]
-   (assert-valid coll-attributes nil))
-  ([coll-attributes options]
-   (let [collection (assoc (dc/collection coll-attributes) :native-id (:native-id coll-attributes))
-         provider-id (get coll-attributes :provider-id "PROV1")
-         response (d/ingest provider-id collection options)]
-     (is (#{{:status 200} {:status 201}} (select-keys response [:status :errors]))))))
-
-(defn assert-valid-keywords
-  [coll-attributes]
-  (assert-valid coll-attributes {:validate-keywords true}))
-
-(defn assert-invalid-spatial
-  ([coord-sys shapes field-path errors]
-   (assert-invalid-spatial coord-sys shapes field-path errors nil))
-  ([coord-sys shapes field-path errors options]
-   (let [shapes (map (partial umm-s/set-coordinate-system coord-sys) shapes)]
-     (assert-invalid {:spatial-coverage (dc/spatial {:gsr coord-sys
-                                                     :sr coord-sys
-                                                     :geometries shapes})}
-                     field-path
-                     errors
-                     options))))
-
-(defn assert-valid-spatial
-  [coord-sys shapes]
-  (let [shapes (map (partial umm-s/set-coordinate-system coord-sys) shapes)]
-    (assert-valid {:spatial-coverage (dc/spatial {:gsr coord-sys
-                                                  :sr coord-sys
-                                                  :geometries shapes})})))
-
-(defn assert-conflict
-  [coll-attributes errors]
-  (let [collection (assoc (dc/collection coll-attributes) :native-id (:native-id coll-attributes))
-        response (d/ingest "PROV1" collection {:allow-failure? true})]
-    (is (= {:status 409
-            :errors errors}
-           (select-keys response [:status :errors])))))
+  (testing "failed validations with no accept or content-type header return xml"
+    (let [concept (assoc (dc/collection-concept {}) :metadata "<Collection>invalid xml</Collection>")
+          {:keys [status body]} (ingest/validate-concept concept {:raw? true})]
+      (is (= [400 "<?xml version=\"1.0\" encoding=\"UTF-8\"?><errors><error>Line 1 - cvc-complex-type.2.3: Element 'Collection' cannot have character [children], because the type's content type is element-only.</error><error>Line 1 - cvc-complex-type.2.4.b: The content of element 'Collection' is not complete. One of '{ShortName}' is expected.</error></errors>"]
+             [status body])))))

 (deftest collection-keyword-validation-test
   ;; For a list of the valid keywords during testing see dev-system/resources/kms_examples
-
   (testing "Keyword validation using validation endpoint"
     (let [concept (dc/collection-concept {:platforms [(dc/platform {:short-name "foo"
                                                                     :long-name "Airbus A340-600"})]})
@@ -327,94 +372,90 @@
           :variable-level-2 "MAXiMUM/MiNiMUM TEMPERATURE"
           :variable-level-3 "24 HOUR MAXiMUM TEMPERATURE"})))

-;; This tests that UMM type validations are applied during collection ingest.
-;; Thorough tests of UMM validations should go in cmr.umm.test.validation.core and related
-;; namespaces.
-;; Currently in the process of moving validation too UMM Spec Lib. Some validation tests
-;; are in cmr.umm-spec.test.validation
 (deftest collection-umm-spec-validation-test
-  (testing "UMM-C JSON-Schema validation"
-    ;; enable return of schema validation errors from API
-    (side/eval-form `(icfg/set-return-umm-json-validation-errors! true))
-    ;; create collection valid against echo10 but invalid against schema
-    (let [response (d/ingest "PROV1" (dc/collection {:product-specific-attributes
-                                                     [(dc/psa {:name "bool1" :data-type :boolean :value true})
-                                                      (dc/psa {:name "bool2" :data-type :boolean :value true})]})
-                             {:allow-failure? true})]
-      (is (= {:status 422
-              :errors ["object has missing required properties ([\"DataCenters\",\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"ScienceKeywords\",\"SpatialExtent\",\"TemporalExtents\"])"]}
-             (select-keys response [:status :errors]))))
-    ;; disable return of schema validation errors from API
-    (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
-    (assert-valid {:product-specific-attributes [(dc/psa {:name "bool1" :data-type :boolean :value true})
-                                                 (dc/psa {:name "bool2" :data-type :boolean :value true})]}))
-
-  ;; testing enabling umm json schema validation through Cmr-Validation-Umm-C header
-  (testing "Enabling UMM-C JSON-Schema validation through Cmr-Validate-Umm-C header being true"
-    ;; create collection valid against echo10 but invalid against schema
-    (let [response (d/ingest "PROV1" (dc/collection {:product-specific-attributes
-                                                     [(dc/psa {:name "bool1" :data-type :boolean :value true})
-                                                      (dc/psa {:name "bool2" :data-type :boolean :value true})]})
-                             {:allow-failure? true :validate-umm-c true})]
-      (is (= {:status 422
-              :errors ["object has missing required properties ([\"DataCenters\",\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"ScienceKeywords\",\"SpatialExtent\",\"TemporalExtents\"])"]}
-             (select-keys response [:status :errors])))))
-
-  (let [coll-attr {:product-specific-attributes
-                   [(dc/psa {:name "bool1" :data-type :boolean :value true})
-                    (dc/psa {:name "bool2" :data-type :boolean :value true})]}]
-    (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
-    (are3 [coll-attributes options]
-          (assert-valid coll-attributes options)
-
-          "Set Cmr-Validate-Umm-C header to false - schema validation error is not returned"
-          coll-attr
-          {:allow-failure? true :validate-umm-c false}
-
-          "Do not set Cmr-Validate-Umm-C header - schema validation error is not returned"
-          coll-attr
-          {:allow-failure? true}))
-
-  ;; testing enabling umm-spec validation through Cmr-Validation-Umm-C header
-  (let [coll-attr {:processing-level-id "1"
-                   :science-keywords [(dc/science-keyword {:category "upcase"
-                                                           :topic "Cool"
-                                                           :term "Mild"})]
-                   :spatial-coverage (dc/spatial {:gsr :cartesian})
-                   :related-urls [(dc/related-url {:type "type" :url "http://www.foo.com"})]
-                   :organizations [{:type :archive-center :org-name "Larc"}]
-                   :platforms [{:short-name "plat"
-                                :long-name "plat"
-                                :type "Aircraft"
-                                :instruments [{:short-name "inst"}]}]
-                   :beginning-date-time "1965-12-12T07:00:00.000-05:00"
-                   :ending-date-time "1967-12-12T07:00:00.000-05:00"
-                   :product-specific-attributes
-                   [(dc/psa {:name "bool" :data-type :boolean :value true})
-                    (dc/psa {:name "bool" :data-type :boolean :value true})]}]
-    (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
-    (side/eval-form `(icfg/set-return-umm-spec-validation-errors! false))
-    (are3 [coll-attributes field-path error options]
-          (assert-invalid coll-attributes field-path error options)
-
-          "Set Cmr-Validate-Umm-C header to true - schema validation passed, umm-spec validation error is returned"
-          coll-attr
-          ["AdditionalAttributes"]
-          ["Additional Attributes must be unique. This contains duplicates named [bool]."]
-          {:validate-umm-c true}
-
-          "Set Cmr-Validate-Umm-C header to false - schema validation passed, umm-spec validation error is not returned"
-          coll-attr
-          ["ProductSpecificAttributes"]
-          ["Product Specific Attributes must be unique. This contains duplicates named [bool]."]
-          {:validate-umm-c false}
-
-          "Do not set Cmr-Validate-Umm-C header - schema validation passed, umm-spec validation error is not returned"
-          coll-attr
-          ["ProductSpecificAttributes"]
-          ["Product Specific Attributes must be unique. This contains duplicates named [bool]."]
-          nil))
-
+  (testing "UMM-C JSON-Schema validation through config settings"
+    (testing "schema validation errors returned"
+      (side/eval-form `(icfg/set-return-umm-json-validation-errors! true))
+      (let [response (d/ingest "PROV1"
+                               (dc/collection {:product-specific-attributes
+                                               [(dc/psa {:name "bool1" :data-type :boolean :value true})
+                                                (dc/psa {:name "bool2" :data-type :boolean :value true})]})
+                               {:allow-failure? true})]
+        (is (= {:status 422
+                :errors ["object has missing required properties ([\"DataCenters\",\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"ScienceKeywords\",\"SpatialExtent\",\"TemporalExtents\"])"]}
+               (select-keys response [:status :errors])))))
+
+    (testing "schema validation errors not returned"
+      (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
+      (assert-valid {:product-specific-attributes [(dc/psa {:name "bool1" :data-type :boolean :value true})
+                                                   (dc/psa {:name "bool2" :data-type :boolean :value true})]})))
+
+  (testing "UMM-C JSON-Schema validation through Cmr-Validate-Umm-C header"
+    (testing "schema validation errors returned when Cmr-Validate-Umm-C header is true"
+      (let [response (d/ingest "PROV1" (dc/collection {:product-specific-attributes
+                                                       [(dc/psa {:name "bool1" :data-type :boolean :value true})
+                                                        (dc/psa {:name "bool2" :data-type :boolean :value true})]})
+                               {:allow-failure? true :validate-umm-c true})]
+        (is (= {:status 422
+                :errors ["object has missing required properties ([\"DataCenters\",\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"ScienceKeywords\",\"SpatialExtent\",\"TemporalExtents\"])"]}
+               (select-keys response [:status :errors])))))
+
+    (testing "schema validation error returns is controlled by config setting when Cmr-Validate-Umm-C header is NOT true"
+      (let [coll-attr {:product-specific-attributes
+                       [(dc/psa {:name "bool1" :data-type :boolean :value true})
+                        (dc/psa {:name "bool2" :data-type :boolean :value true})]}]
+        (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
+        (are3 [coll-attributes options]
+              (assert-valid coll-attributes options)
+
+              "Set Cmr-Validate-Umm-C header to false - schema validation error is not returned"
+              coll-attr
+              {:allow-failure? true :validate-umm-c false}
+
+              "Do not set Cmr-Validate-Umm-C header - schema validation error is not returned"
+              coll-attr
+              {:allow-failure? true}))))
+
+  (testing "UMM-SPEC validation through Cmr-Validation-Umm-C header"
+    (let [coll-attr {:processing-level-id "1"
+                     :science-keywords [(dc/science-keyword {:category "upcase"
+                                                             :topic "Cool"
+                                                             :term "Mild"})]
+                     :spatial-coverage (dc/spatial {:gsr :cartesian})
+                     :related-urls [(dc/related-url {:type "type" :url "http://www.foo.com"})]
+                     :organizations [{:type :archive-center :org-name "Larc"}]
+                     :platforms [{:short-name "plat"
+                                  :long-name "plat"
+                                  :type "Aircraft"
+                                  :instruments [{:short-name "inst"}]}]
+                     :beginning-date-time "1965-12-12T07:00:00.000-05:00"
+                     :ending-date-time "1967-12-12T07:00:00.000-05:00"
+                     :product-specific-attributes
+                     [(dc/psa {:name "bool" :data-type :boolean :value true})
+                      (dc/psa {:name "bool" :data-type :boolean :value true})]}]
+      (side/eval-form `(icfg/set-return-umm-json-validation-errors! false))
+      (side/eval-form `(icfg/set-return-umm-spec-validation-errors! false))
+      (are3 [coll-attributes field-path error options]
+            (assert-invalid coll-attributes field-path error options)
+
+            "Set Cmr-Validate-Umm-C header to true - schema validation passed, umm-spec validation error is returned"
+            coll-attr
+            ["AdditionalAttributes"]
+            ["Additional Attributes must be unique. This contains duplicates named [bool]."]
+            {:validate-umm-c true}
+
+            "Set Cmr-Validate-Umm-C header to false - schema validation passed, umm-spec validation error is not returned"
+            coll-attr
+            ["ProductSpecificAttributes"]
+            ["Product Specific Attributes must be unique. This contains duplicates named [bool]."]
+            {:validate-umm-c false}
+
+            "Do not set Cmr-Validate-Umm-C header - schema validation passed, umm-spec validation error is not returned"
+            coll-attr
+            ["ProductSpecificAttributes"]
+            ["Product Specific Attributes must be unique. This contains duplicates named [bool]."]
+            nil)))
+
   (side/eval-form `(icfg/set-return-umm-spec-validation-errors! true))

   (testing "Additional Attribute validation"
@@ -424,12 +465,14 @@
         (dc/psa {:name "bool" :data-type :boolean :value true})]}
       ["AdditionalAttributes"]
       ["Additional Attributes must be unique. This contains duplicates named [bool]."]))
+
   (testing "Nested Path Validation"
     (assert-invalid
       {:platforms [(dc/platform {:instruments [(dc/instrument {:short-name "I1"})
                                                (dc/instrument {:short-name "I1"})]})]}
       ["Platforms" 0 "Instruments"]
       ["Instruments must be unique. This contains duplicates named [I1]."]))
+
   (testing "Spatial validation"
     (testing "geodetic polygon"
       ;; Invalid points are caught in the schema validation
@@ -441,6 +484,7 @@
          "Spatial validation error: The shape contained duplicate points. Points 3 [lon=-180 lat=-90] and 4 [lon=180 lat=-90] were considered equivalent or very close."
          "Spatial validation error: The shape contained consecutive antipodal points. Points 2 [lon=-180 lat=90] and 3 [lon=-180 lat=-90] are antipodal."
          "Spatial validation error: The shape contained consecutive antipodal points. Points 4 [lon=180 lat=-90] and 5 [lon=180 lat=90] are antipodal."]))
+
     (testing "geodetic polygon with holes"
       ;; Hole validation is not supported yet. See CMR-1173.
       ;; Holes are ignored during validation for now.
@@ -481,12 +525,13 @@
       {:entry-title "ET-1" :concept-id "C1-PROV1" :native-id "Native1"})
     (assert-valid
       {:entry-title "ET-1" :concept-id "C1-PROV2" :native-id "Native1" :provider-id "PROV2"}))
+
   (testing "entry-title must be unique for a provider"
     (assert-conflict
       {:entry-title "ET-1" :concept-id "C2-PROV1" :native-id "Native2"}
       ["The Entry Title [ET-1] must be unique. The following concepts with the same entry title were found: [C1-PROV1]."])))

-(deftest header-validations
+(deftest revision-id-validations
   (testing "ingesting a concept with the same concept-id and revision-id fails"
     (let [concept-id "C1-PROV1"
           existing-concept (dc/collection-concept {:revision-id 1 :concept-id concept-id})
@@ -495,22 +540,27 @@
       (is (= {:status 409
               :errors [(format "Expected revision-id of [2] got [1] for [%s]" concept-id)]}
              response))))
+
   (testing "attempting to ingest using an non-integer revision id returns an error"
     (let [response (ingest/ingest-concept (dc/collection-concept {:concept-id "C2-PROV1"
                                                                   :revision-id "NaN"}))]
       (is (= {:status 422
               :errors [(msg/invalid-revision-id "NaN")]}
              response))))
+
   (testing "attempting to ingest using a negative revision id returns an error"
     (let [response (ingest/ingest-concept (dc/collection-concept {:concept-id "C2-PROV1"
                                                                   :revision-id "-1"}))]
       (is (= {:status 422
               :errors [(msg/invalid-revision-id "-1")]}
              response))))
+
   (testing "ingesting a concept with just the revision-id succeeds"
     (let [response (ingest/ingest-concept (dc/collection-concept {:revision-id "2"}))]
       (is (and (= 200 (:status response)) (= 2 (:revision-id response))))))
-  (testing "ingesting a concept while skipping revision-ids succeeds, but fails if revision id is smaller than the maximum revision id"
+
+  (testing "ingesting a concept while skipping revision-ids succeeds,
+            but fails if revision id is smaller than the maximum revision id"
     (let [concept-id "C3-PROV1"
           coll (dc/collection-concept {:concept-id concept-id})
           _ (ingest/ingest-concept (assoc coll :revision-id "2"))
@@ -521,19 +571,14 @@
               :errors [(format "Expected revision-id of [7] got [4] for [%s]" concept-id)]}
              response2)))))

-(defn- assert-revision-conflict
-  [concept-id format-str response]
-  (is (= {:status 409
-          :errors [(format format-str concept-id)]}
-         response)))
-
 ;; Added to test out of order processing of ingest and delete requests with revision-ids in their
 ;; header. The proper handling of incorrectly ordered requests is important for Virtual Product
 ;; Service which picks events off the queue and sends them to ingest service. It cannot be
-;; gauranteed that the ingest events are processed by Virtual Product Service in the same order
+;; guaranteed that the ingest events are processed by Virtual Product Service in the same order
 ;; that the events are placed on the queue.
 (deftest revision-conflict-tests
-  (testing "Update with lower revision id should be rejected if it comes after an concept with a higher revision id"
+  (testing "Update with lower revision id should be rejected
+            if it comes after an concept with a higher revision id"
     (let [concept (dc/collection-concept {:revision-id 4})
           concept-id (:concept-id (ingest/ingest-concept concept))
           response (ingest/ingest-concept (assoc concept :revision-id 2))]
@@ -577,36 +622,37 @@
              response)))))

 (deftest dif-to-echo10-conversion
-  (let [coll (d/ingest "PROV1"
-                       (dc/collection-dif {:short-name "S1"
-                                           :version-id "V1"
-                                           :entry-title "ET1"
-                                           :long-name "L4"
-                                           :summary (name :dif)
-                                           ;; The following fields are needed for DIF to pass xml validation
-                                           :science-keywords [(dc/science-keyword {:category "upcase"
-                                                                                :topic "Cool"
-                                                                                :term "Mild"})]
-                                           :organizations [(dc/org :distribution-center "Larc")]
-                                           ;; The following fields are needed for DIF10 to pass xml validation
-                                           :beginning-date-time "1965-12-12T12:00:00Z"
-                                           :ending-date-time "1967-12-12T12:00:00Z"
-                                           :product-specific-attributes [(c/map->ProductSpecificAttribute {:name "moo" :data-type :string})]})
-                       {:format :dif})
-        response (search/retrieve-concept (:concept-id coll) nil {:accept mt/echo10})
-        concept-map {:metadata (:body response)
-                     :format mt/echo10
-                     :concept-type :collection
-                     :provider-id "PROV1"
-                     :native-id "new collection"}
-        {:keys [status body]} (ingest/validate-concept concept-map {:raw? true})]
-    (is (= 200 status))))
+  (testing "Conversion from dif9 to echo10 is successful"
+    (let [coll (d/ingest "PROV1"
+                         (dc/collection-dif
+                           {:short-name "S1"
+                            :version-id "V1"
+                            :entry-title "ET1"
+                            :long-name "L4"
+                            :summary (name :dif)
+                            ;; The following fields are needed for DIF to pass xml validation
+                            :science-keywords [(dc/science-keyword {:category "upcase"
+                                                                    :topic "Cool"
+                                                                    :term "Mild"})]
+                            :organizations [(dc/org :distribution-center "Larc")]
+                            ;; The following fields are needed for DIF10 to pass xml validation
+                            :beginning-date-time "1965-12-12T12:00:00Z"
+                            :ending-date-time "1967-12-12T12:00:00Z"
+                            :product-specific-attributes
+                             [(c/map->ProductSpecificAttribute {:name "moo" :data-type :string})]})
+                         {:format :dif})
+          response (search/retrieve-concept (:concept-id coll) nil {:accept mt/echo10})
+          concept-map {:metadata (:body response)
+                       :format mt/echo10
+                       :concept-type :collection
+                       :provider-id "PROV1"
+                       :native-id "new collection"}
+          {:keys [status body]} (ingest/validate-concept concept-map {:raw? true})]
+      (is (= 200 status)))))

 (deftest umm-spec-validation-warnings
-  ;; By default the config return-umm-spec-validation-errors is false, so warnings are returned
-  ;; with the collection. If return-umm-spec-validation-errors was true, errors would be thrown. tests
-  ;; for the errors if return-umm-spec-validation-errors is true are in this file.
-  (testing "Ingest and Ingest Validation"
+  ;; By default the config return-umm-spec-validation-errors is false, so warnings are returned with the collection.
+  (testing "Ingest and Ingest Validation with warning messages for all formats"
     (are3 [format collection warning-message]
           (do
             (let [response (d/ingest "PROV1" collection {:format format})]
@@ -633,22 +679,20 @@

           "DIF9 with no version - has warnings, but passes ingest"
           :dif (assoc-in (dc/collection-dif {}) [:product :version-id] nil)
-          "After translating item to UMM-C the metadata had the following issue: object has missing required properties ([\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"SpatialExtent\",\"TemporalExtents\",\"Version\"])"))
-
-  (testing "Multiple warnings"
-     (let [collection (dc/collection-dif10 {:platforms [(dc/platform {:short-name (apply str (repeat 81 "x"))})]
-                                            :purpose (apply str (repeat 12000 "y"))
-                                            :product (dc/product {:processing-level-id "1"})})
-           ingest-response (d/ingest "PROV1" collection {:format :dif10})
-           validation-response (ingest/validate-concept (dc/collection-concept collection :dif10))]
-       (is (some? (re-find #"/Platforms/0/ShortName string.*is too long \(length: 81, maximum allowed: 80\)" (:warnings ingest-response))))
-       (is (some? (re-find #"/Platforms/0/ShortName string.*is too long \(length: 81, maximum allowed: 80\)" (:warnings validation-response))))
-       (is (some? (re-find #"/Purpose string.*is too long \(length: 12000, maximum allowed: 10000\)" (:warnings ingest-response))))
-       (is (some? (re-find #"/Purpose string.*is too long \(length: 12000, maximum allowed: 10000\)" (:warnings validation-response))))))
-  (testing "Warnings returned in JSON format"
-     (let [response (d/ingest "PROV1" (dc/collection-dif10 {}) {:format :dif10 :accept-format :json})]
-       (is (= 201 (:status response)))
-       (is (= ["After translating item to UMM-C the metadata had the following issue: object has missing required properties ([\"ProcessingLevel\"])"] (:warnings response))))))
+          "After translating item to UMM-C the metadata had the following issue: object has missing required properties ([\"Platforms\",\"ProcessingLevel\",\"RelatedUrls\",\"SpatialExtent\",\"TemporalExtents\",\"Version\"])")))
+
+(deftest field-exceeding-maxlength-warnings
+  (testing "Multiple warnings returned for the fields exceeding maxlength allowed"
+    (let [collection (dc/collection-dif10
+                       {:platforms [(dc/platform {:short-name (apply str (repeat 81 "x"))})]
+                        :purpose (apply str (repeat 12000 "y"))
+                        :product (dc/product {:processing-level-id "1"})})
+          ingest-response (d/ingest "PROV1" collection {:format :dif10})
+          validation-response (ingest/validate-concept (dc/collection-concept collection :dif10))]
+      (is (some? (re-find #"/Platforms/0/ShortName string.*is too long \(length: 81, maximum allowed: 80\)" (:warnings ingest-response))))
+      (is (some? (re-find #"/Platforms/0/ShortName string.*is too long \(length: 81, maximum allowed: 80\)" (:warnings validation-response))))
+      (is (some? (re-find #"/Purpose string.*is too long \(length: 12000, maximum allowed: 10000\)" (:warnings ingest-response))))
+      (is (some? (re-find #"/Purpose string.*is too long \(length: 12000, maximum allowed: 10000\)" (:warnings validation-response)))))))

 (comment
   (ingest/delete-provider "PROV1")


Add more type tests for IPersistentMap wrappers.

--- test/guangyin/internal/types_test.clj
+++ test/guangyin/internal/types_test.clj
@@ -47,7 +47,11 @@
     (is (= month (unwrap wrapped)))
     (is (= 6 (:month-of-year wrapped)))
     (is (= nil (:year wrapped)))
-    (is (= 42 (get wrapped :year 42)))))
+    (is (= 42 (get wrapped :year 42)))
+    (is (= true (contains? wrapped :month-of-year)))
+    (is (= false (contains? wrapped :year)))
+    (is (= :month-of-year (.key (find wrapped :month-of-year))))
+    (is (= 6 (.val (find wrapped :month-of-year))))))

 (deftest test-duration
   (let [duration (java.time.Duration/parse "PT1H2M3S")
@@ -61,7 +65,11 @@
     (is (= 3723 (:seconds wrapped)))
     (is (= 3723000 (:millis wrapped)))
     (is (= 3723000000 (:micros wrapped)))
-    (is (= 3723000000000 (:nanos wrapped)))))
+    (is (= 3723000000000 (:nanos wrapped)))
+    (is (= true (contains? wrapped :hours)))
+    (is (= false (contains? wrapped :days)))
+    (is (= :hours (.key (find wrapped :hours))))
+    (is (= 1 (.val (find wrapped :hours))))))

 (deftest test-temporal-amount
   (let [period (java.time.Period/parse "P1Y2M3D")
@@ -72,4 +80,9 @@
     (is (= 1 (:years wrapped)))
     (is (= 2 (:months wrapped)))
     (is (= 3 (:days wrapped)))
-    (is (= nil (:hours wrapped)))))
+    (is (= nil (:hours wrapped)))
+    (is (= true (contains? wrapped :years)))
+    (is (= false (contains? wrapped :hours)))
+    (is (= :years (.key (find wrapped :years))))
+    (is (= 1 (.val (find wrapped :years))))))
+


Remove large file test, circleci can't handle it


Add dedupe on state persistence stream.

Add dedupe on state persistence stream.

--- frontend/uxbox/core.cljs
+++ frontend/uxbox/core.cljs
@@ -18,6 +18,7 @@
                              :projects-by-id])
         stream (->> (l/focus-atom lens st/state)
                     (rx/from-atom)
+                    (rx/dedupe)
                     (rx/debounce 1000)
                     (rx/tap #(println "[save]")))]
     (rx/on-value stream #(dl/persist-state %))))


add contact and projects section

add contact and projects section

--- app/src/cljs/app/res.cljs
+++ app/src/cljs/app/res.cljs
@@ -1,4 +1,10 @@
 (ns app.res)

 (def resume
-  {:profile "Martin Haesler is a former sales man for plumbing and heating projects, who after 17 years of working in this line of work, decided it was time for a change, see something of the world, and try for a completely different career path. Having always been interested in computer technology, an avid Linux user for many years, he decided on programming as a new career.\nMartin Haesler is quick to learn new things, self motivated and eager to improve."})
+  {:profile "Martin Haesler is a former sales man for plumbing and heating projects, who after 17 years of working in this line of work, decided it was time for a change, see something of the world, and try for a completely different career path. Having always been interested in computer technology, an avid Linux user for many years, he decided on programming as a new career.\nMartin Haesler is quick to learn new things, self motivated and eager to improve."
+   :contact {:name "Martin Haesler"
+             :email "martin.haesler@gmail.com"
+             :phone "07481 598020"}
+   :projects {:title "ThemeCreator"
+              :url "https://github.com/mswift42/themecreator"
+              :description "ThemeCreator lets you easily create beautiful color themes for IntelliJ Editors, Atom, Textmate and Emacs.\nThemeCreator is a Clojurescript app, built using the reagent framework."}})


Logging function that can be used in threads.

Sort of like a tap, but specifically for logging.

--- src/asid/error/thread.clj
+++ src/asid/error/thread.clj
@@ -2,7 +2,8 @@
   (:use midje.sweet
         asid.error.definition)

-  (:require [clojure.algo.monads :as monad])
+  (:require [clojure.algo.monads :as monad]
+            [clojure.tools.logging :as log])

   (:import [asid.error.definition Failure]))

@@ -83,3 +84,8 @@
 (fact
   (more-steps 10) => 100
   (more-steps 100) => (partial instance? Failure))
+
+(defn -log-> [obj msg]
+  (log/debug msg)
+  (log/debug obj)
+  obj)


Simpler line in simple/core

Simpler line in simple/core

It also makes the code like in the walkthrough:
<https://github.com/Day8/re-frame/blob/master/docs/CodeWalkthrough.md#reg-sub>

--- examples/simple/src/simple/core.cljs
+++ examples/simple/src/simple/core.cljs
@@ -44,8 +44,7 @@
 (rf/reg-sub
   :time
   (fn [db _]     ;; db is current app state. 2nd unused param is query vector
-    (-> db
-        :time)))
+    (:time db))) ;; return a query computation over the application state

 (rf/reg-sub
   :time-color


Add delete string.

Add delete string.

--- src/beebster_clj/core.clj
+++ src/beebster_clj/core.clj
@@ -1,29 +1,26 @@
 (ns beebster-clj.core
   (:require [compojure.route :as route]
             [compojure.handler :as handler]
-            [beebster-clj.views :as views])
+            ;; [beebster-clj.views :as views]
+            )
   (:use [ring.adapter.jetty :only [run-jetty]]
         [clojure.java.shell :only [sh]]
         [clojure.string :as str :only [split-lines ]]
-        hiccup.core hiccup.util hiccup.page compojure.core))
+        hiccup.core hiccup.util hiccup.page compojure.core ))

 (def iplayer-command
   '("get_iplayer" "--nocopyright" "--limitmatches" "50" "--listformat" "<index> <pid> <thumbnail> <name> <episode>"))

-(defn iplayer-search
-  "run external command with given args list."
-  [search-title]
-  (split-lines (:out (apply sh (flatten
-                                (conj (list search-title) iplayer-command))))))
+

 (def iplayer-info
   '("get_iplayer" "-i"))

-(defn old-recordings?
-  "Does get-iplayer complain about recorded programmes > 30 days?"
-  [str]
-  (remove nil? (map #(re-find #"These programmes should be deleted:" %) str)))
+(def delete-string
+  "These programmes should be deleted:")
+
+

 (defn get-info
   "get-iplayer info for entered index"
@@ -46,19 +43,45 @@
   [s]
   (remove #(= "" %) (map #(re-find #"^[0-9]*" %) s)))

+(defn old-recordings?
+  "Does get-iplayer complain about recorded programmes > 30 days?"
+  [str]
+  (remove nil? (map #(re-find #"These programmes should be deleted:" %) str)))

 (defn iplayer-download-command
   "concatenate index and mode to download command"
    [index mode]
   (apply str  "get_iplayer " mode "1" " -g  --nocopyright --output=\"$HOME/Videos\"" " " index))

+(defn get-url
+  "return /result url string concatenated with index"
+  [index]
+  (apply str "/result?index=" index))
+
+(defn iplayer-search-resultstring
+  "run external command with given args list."
+  [search-title]
+  (:out (apply sh (flatten
+                   (conj (list search-title) iplayer-command)))))
+
+
+
+(defn iplayer-search
+  "run external command with given args list."
+  [search-title]
+  (let [result (iplayer-search-resultstring search-title)]
+    (if (old-recordings? (split-lines result))
+      (split-lines (subs result (.indexOf result delete-string)))
+      (split-lines result))))
+

 (defn search-categories
   "search iplayer for a given category"
   [cat]
-  (split-lines (:out (apply sh (flatten
-                                (conj (list "--category" cat)
-                                      iplayer-command))))))
+  (drop 1 (butlast
+           (split-lines (:out (apply sh (flatten
+                                         (conj (list "--category" cat)
+                                               iplayer-command))))))))


[GAWB-1623] Method Config Edit: Cancel Editing does not restore origi… (#812)

* [GAWB-1623] Method Config Edit: Cancel Editing does not restore original Snapshot ID

* Code review comments

--- src/cljs/main/broadfcui/page/workspace/method_configs/method_config_editor.cljs
+++ src/cljs/main/broadfcui/page/workspace/method_configs/method_config_editor.cljs
@@ -95,21 +95,22 @@
                      (swap! state assoc :error status-text)))}))})

-(defn- render-side-bar [state refs config editing? props]
+(defn- render-side-bar [{:keys [state refs config editing? props restore-on-cancel]}]
   [:div {:style {:width 290 :float "left"}}
    [:div {:ref "sidebar"}]
    (style/create-unselectable :div {:style {:position (when-not (:sidebar-visible? @state) "fixed")
                                             :top (when-not (:sidebar-visible? @state) 4)
                                             :width 290}}
      (let [locked? (:locked? @state)
-           can-edit? (access-greater-than? (:access-level props) "READER")]
+           can-edit? (access-greater-than? (:access-level props) "READER")
+           snapshot-id (get-in config ["methodRepoMethod" "methodVersion"])]
        [:div {:style {:lineHeight 1}}
         (when (and can-edit? (not editing?))
           [:div {}
            [comps/SidebarButton {:style :light :color :button-primary
                                  :text "Edit Configuration" :icon :edit
                                  :disabled? (when locked? "The workspace is locked")
-                                 :onClick #(swap! state assoc :editing? true)}]
+                                 :onClick #(swap! state assoc :editing? true :prev-snapshot-id snapshot-id)}]
            [comps/SidebarButton {:style :light :color :exception-state :margin :top
                                  :text "Delete" :icon :delete
                                  :disabled? (when locked? "The workspace is locked")
@@ -128,12 +129,15 @@
         (when editing?
           [comps/SidebarButton {:color :success-state
                                 :text "Save" :icon :done
-                                :onClick #(do (commit state refs config props)
-                                              (stop-editing state))}])
+                                :onClick (fn []
+                                           (commit state refs config props)
+                                           (stop-editing state))}])
         (when editing?
           [comps/SidebarButton {:color :exception-state :margin :top
                                 :text "Cancel Editing" :icon :cancel
-                                :onClick #(stop-editing state)}])]))])
+                                :onClick (fn []
+                                           (restore-on-cancel (:prev-snapshot-id @state))
+                                           (stop-editing state))}])]))])

 (defn- input-output-list [values ref-prefix invalid-values editing? all-values]
@@ -212,11 +216,12 @@
 (defn- render-display [this state refs props]
   (let [wrapped-config (:loaded-config @state)
         config (wrapped-config "methodConfiguration")
-        editing? (:editing? @state)]
+        editing? (:editing? @state)
+        restore-on-cancel #(this :load-new-method-template %)]
     [:div {}
      [comps/Blocker {:banner (:blocker @state)}]
      [:div {:style {:padding "1em 2em"}}
-      (render-side-bar state refs config editing? props)
+      (render-side-bar (utils/restructure state refs config editing? props restore-on-cancel))
       (when-not editing?
         [:div {:style {:float "right"}}
          (launch/render-button {:workspace-id (:workspace-id props)


pass input to step function

pass input to step function

--- figwheel-browser/src/zajal/core.cljs
+++ figwheel-browser/src/zajal/core.cljs
@@ -3,12 +3,36 @@

 (enable-console-print!)

-(defn sketch [start update draw]
+(defonce start-time (js/Date.now))
+
+(defonce mouse-state (atom {:x 0 :y 0 :pressed false}))
+
+(set! js/document.onmousemove
+      #(swap! mouse-state
+             assoc
+             :x (.-pageX %)
+             :y (.-pageY %)))
+
+(set! js/document.onmousedown
+      #(swap! mouse-state
+              assoc :pressed? true))
+
+(set! js/document.onmouseup
+      #(swap! mouse-state
+              assoc :pressed? false))
+
+(defn input []
+  (let [t (js/Date.now)]
+    {:time {:elapsed (- t start-time)}
+     :mouse @mouse-state }))
+
+(defn sketch [start step draw]
+  (step {} (input))
   (let [state (reagent/atom start)]
     (letfn [(component [] [draw @state])
             (render-loop [t]
                          (.begin js/stats)
-                         (swap! state update)
+                         (swap! state step (input))
                          (.end js/stats)
                          (.requestAnimationFrame js/window render-loop))]
       (reagent/render-component


Transact schema in database start.

Transact schema in database start.

--- src/clj/re_recipes/db.clj
+++ src/clj/re_recipes/db.clj
@@ -8,6 +8,7 @@
   (start [component]
     (d/create-database uri)
     (let [conn (d/connect uri)]
+      @(d/transact conn schema)
       (assoc component :db conn)))

   (stop [component]


tiny bugfix

tiny bugfix

--- src/clj/tictag/server.clj
+++ src/clj/tictag/server.clj
@@ -162,10 +162,10 @@
     (do
       (debugf "Received a timestamp: %s" (pr-str params))
       (update-pings! component user [(-> params
-                                  (assoc :user-id (:id user))
-                                  (update :timestamp cli/str-number?)
-                                  (dissoc :username :password))])
-      (response {:status :ok}))
+                                         (assoc :user-id (:id user))
+                                         (update :timestamp cli/str-number?)
+                                         (dissoc :username :password))])
+      (response {:status 200}))
     UNAUTHORIZED))

 (defn health-check [component]


Update state shape

Update state shape

--- src/cljs/gtfve/state.cljs
+++ src/cljs/gtfve/state.cljs
@@ -14,7 +14,8 @@
                         :last-query ""
                         :loading? false}}}
    :data {:stops-search-results []
-          :stops []
+          :changes []
+          :stops {}
           :stop-times []
           :routes []}})


Docstring correction in replumb.ast/dissoc-all

--- src/cljs/replumb/ast.cljs
+++ src/cljs/replumb/ast.cljs
@@ -218,7 +218,7 @@
 (defn dissoc-all
   "Dissoc all the required-ns symbols from requirer-ns.

-  There are three types of symbol in replumb jargon, which loosely map to
+  There are five types of symbol in replumb jargon, which loosely map to
   `cljs.js` ones. These optionally go in the type parameter as keyword:

   - `:symbol`, the default, for instance my-sym in `(def my-sym 3)`


still on chapter 2of brave clojure

still on chapter 2of brave clojure

--- src/clojure_noon/core.clj
+++ src/clojure_noon/core.clj
@@ -11,4 +11,41 @@
 (defn train [] (println "Choo Choo!"))

 (println "Cleanliness is the next to godliness")
-(map)
+
+;wrapping and slurping in emacs
+(+ 1 (* 2) 3 4)
+
+(if true
+  "By Zeus's hammer!"
+  "By Aquama's trident!")
+
+(if false
+  "By Zeus's hammer!"
+  "By Aquaman's trident!")
+
+(if false
+  "By Odin's Elbow")
+
+(if true
+  (do (println "Success!")
+      "By Zeus's hammer!")
+  (do (println "Failure!")
+      "By Aquaman's trident!"))
+
+(when true
+  (println "Success!")
+  "abra cadabra")
+
+(nil? 1)
+
+(nil? nil)
+
+(if "bears eats beets"
+  "bears beets Battlestar Galactica")
+
+(if nil
+  "This won't be the result because nil is false"
+  "nil is falsey")
+
+(= 1 1)
+


Updating less to latest bidi

Updating less to latest bidi

--- modules/less/project.clj
+++ modules/less/project.clj
@@ -1,18 +1,16 @@
 ;; Copyright © 2014 JUXT LTD.

-(defproject juxt.modular/less "0.1.1"
+(defproject juxt.modular/less "0.1.2"
   :description "A modular extension that lets you compile Less files into CSS"
   :url "https://github.com/juxt/modular/tree/master/modules/less"
   :license {:name "The MIT License"
             :url "http://opensource.org/licenses/MIT"}
-  :dependencies [
-                 [prismatic/schema "0.3.3"]
+  :dependencies [[prismatic/schema "0.3.3"]
                  [com.stuartsierra/component "0.2.2"]
                  [malcolmsparks/clj-less "1.7.3"]
-                 [juxt.modular/bidi "0.6.1"]]
+                 [juxt.modular/bidi "0.7.1"]]

   :profiles {:dev {:dependencies
                    [[org.clojure/clojure "1.6.0"]
                     [org.webjars/bootstrap "3.3.0"]
-                    [juxt.modular/test "0.1.0"]]}
-             })
+                    [juxt.modular/test "0.1.0"]]}})


Fixed kibit error in elastic-utils.

Fixed kibit error in elastic-utils.

--- elastic-utils-lib/src/cmr/elastic_utils/connect.clj
+++ elastic-utils-lib/src/cmr/elastic_utils/connect.clj
@@ -77,5 +77,5 @@
   "Returns the elasticsearch health with timeout handling."
   [context elastic-key-in-context]
   (let [;; We add 1 second to allow get-elastic-health operation to timeout first
-        timeout-ms (* 1000 (+ 1 (hh/health-check-timeout-seconds)))]
+        timeout-ms (* 1000 (inc (hh/health-check-timeout-seconds)))]
     (hh/get-health #(health-fn context elastic-key-in-context) timeout-ms)))


CLJS-963: do not bother computing goog/dep.js under :none

Previously this would be computed to include only what the user had
required from Closure Library. This is unnecessary under optimizations
:none and when rebuilding source this will often not get properly emitted.
Instead just copy it over.


Just a little palette for the kids.

Just a little palette for the kids.

Took me 5m to code.


file based bash invocation

file based bash invocation

--- src/flax/node_manager.clj
+++ src/flax/node_manager.clj
@@ -17,38 +17,44 @@
   (invoke [self checkpoint]
     (if-not (:invocation checkpoint)
       checkpoint
-      (let [argv (remove nil?
+      (let [tmpfile-name (str ".flax-temp-script-" (java.util.UUID/randomUUID))
+            argv (remove nil?
                    (flatten
-                     [(if-let [u (:user checkpoint)] ["sudo" "-u" u]) "bash" "-c" (:invocation checkpoint)]))
-            proc (-> (Runtime/getRuntime)
-                   (.exec (into-array String argv)))
-            stdout (stream-to-reader (.getInputStream proc))
-            stderr (stream-to-reader (.getErrorStream proc))]
-        (when-not (false? (:log checkpoint))
-          (log logger :info (clojure.string/join " " argv)))
-        (loop [out []
-               err []]
-          (let [line (.readLine stdout)
-                errl (.readLine stderr)]
-            (when-not (false? (:log checkpoint))
-              (when-not (clojure.string/blank? line)
-                (log logger :debug line))
-              (when-not (clojure.string/blank? errl)
-                (log logger :debug errl)))
-            (if-not (and (nil? line)
-                         (nil? errl))
-              (recur (if-not (nil? line)
-                       (conj out line) out)
-                     (if-not (nil? errl)
-                       (conj err errl) err))
-              (let [out (clojure.string/join "\n" out)
-                    err (clojure.string/join "\n" err)]
-                (assoc checkpoint :out {:keys (:out checkpoint)
-                                        :value out}
-                                  :err {:keys (:err checkpoint)
-                                        :value err}
-                                  :exit (when (:throw checkpoint)
-                                          (.waitFor proc))))))))))
+                     [(if-let [u (:user checkpoint)] ["sudo" "-u" u]) "bash" tmpfile-name]))]
+        (spit tmpfile-name (:invocation checkpoint))
+        (let [proc (-> (Runtime/getRuntime)
+                     (.exec (into-array String argv)))
+              stdout (stream-to-reader (.getInputStream proc))
+              stderr (stream-to-reader (.getErrorStream proc))]
+          (future
+            (Thread/sleep 5000)
+            (clojure.java.io/delete-file tmpfile-name))
+          (when-not (false? (:log checkpoint))
+            (log logger :info (clojure.string/join " " argv))
+            (log logger :info (str "Contents of " tmpfile-name ":\n" (:invocation checkpoint))))
+          (loop [out []
+                 err []]
+            (let [line (.readLine stdout)
+                  errl (.readLine stderr)]
+              (when-not (false? (:log checkpoint))
+                (when-not (clojure.string/blank? line)
+                  (log logger :debug line))
+                (when-not (clojure.string/blank? errl)
+                  (log logger :debug errl)))
+              (if-not (and (nil? line)
+                           (nil? errl))
+                (recur (if-not (nil? line)
+                         (conj out line) out)
+                       (if-not (nil? errl)
+                         (conj err errl) err))
+                (let [out (clojure.string/join "\n" out)
+                      err (clojure.string/join "\n" err)]
+                  (assoc checkpoint :out {:keys (:out checkpoint)
+                                          :value out}
+                                    :err {:keys (:err checkpoint)
+                                          :value err}
+                                    :exit (when (:throw checkpoint)
+                                            (.waitFor proc)))))))))))

   (clone [self]
     self))


Fix password-reset url. Closes #164.

Fix password-reset url. Closes #164.

--- src/foreclojure/login.clj
+++ src/foreclojure/login.clj
@@ -10,6 +10,8 @@
             [clojail.core             :only [thunk-timeout]]
             [clojure.stacktrace       :only [print-cause-trace]]
             [somnium.congomongo       :only [update! fetch-one]]))
+
+(def password-reset-url "https://www.4clojure.com/settings")

 (def login-box
   (form-to [:post "/login"]
@@ -85,7 +87,7 @@
            :text
            (str "The password for your 4clojure.com account "
                 name " has been reset to " pw ". Make sure to change it"
-                " soon at https://4clojure.com/login/update - pick"
+                " soon at " password-reset-url " - pick"
                 " something you'll remember!")})
          {:success true})
        10 :sec)
@@ -103,8 +105,8 @@
                                             :only [:_id :user])]
     (let [{:keys [success] :as diagnostics} (try-to-email email name id)]
       (if success
-        (do (session/session-put! :login-to "/login/update")
-            (flash-msg (login-url "/login/update")
+        (do (session/session-put! :login-to password-reset-url)
+            (flash-msg (login-url password-reset-url)
               "Your password has been reset! You should receive an email soon."))
         (do (spit (str name ".pwd") diagnostics)
             (flash-error "/login/reset"


add example tab snippet

add example tab snippet


:bug: Add proper constraint to integer values on file specs.

--- common/app/common/pages.cljc
+++ common/app/common/pages.cljc
@@ -20,8 +20,9 @@
    [app.common.spec :as us]
    [app.common.uuid :as uuid]))

-(def page-version 5)
 (def file-version 1)
+(def max-safe-int 9007199254740991)
+(def min-safe-int -9007199254740991)

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Page Transformation Changes
@@ -38,9 +39,13 @@
 (s/def ::string string?)
 (s/def ::type keyword?)
 (s/def ::uuid uuid?)
+(s/def ::safe-integer
+  #(and
+    (integer? %)
+    (>= % min-safe-int)
+    (<= % max-safe-int)))

 ;; Page Options
-

 ;; TODO: missing specs for :saved-grids

@@ -70,43 +75,43 @@
 (s/def :internal.shape/fill-color string?)
 (s/def :internal.shape/fill-color-ref-file (s/nilable uuid?))
 (s/def :internal.shape/fill-color-ref-id (s/nilable uuid?))
-(s/def :internal.shape/fill-opacity number?)
+(s/def :internal.shape/fill-opacity ::safe-integer)
 (s/def :internal.shape/font-family string?)
-(s/def :internal.shape/font-size number?)
+(s/def :internal.shape/font-size ::safe-integer)
 (s/def :internal.shape/font-style string?)
 (s/def :internal.shape/font-weight string?)
 (s/def :internal.shape/hidden boolean?)
-(s/def :internal.shape/letter-spacing number?)
-(s/def :internal.shape/line-height number?)
+(s/def :internal.shape/letter-spacing ::safe-integer)
+(s/def :internal.shape/line-height ::safe-integer)
 (s/def :internal.shape/locked boolean?)
 (s/def :internal.shape/page-id uuid?)
-(s/def :internal.shape/proportion number?)
+(s/def :internal.shape/proportion ::safe-integer)
 (s/def :internal.shape/proportion-lock boolean?)
-(s/def :internal.shape/rx number?)
-(s/def :internal.shape/ry number?)
+(s/def :internal.shape/rx ::safe-integer)
+(s/def :internal.shape/ry ::safe-integer)
 (s/def :internal.shape/stroke-color string?)
 (s/def :internal.shape/stroke-color-ref-file (s/nilable uuid?))
 (s/def :internal.shape/stroke-color-ref-id (s/nilable uuid?))
-(s/def :internal.shape/stroke-opacity number?)
+(s/def :internal.shape/stroke-opacity ::safe-integer)
 (s/def :internal.shape/stroke-style #{:solid :dotted :dashed :mixed :none})
-(s/def :internal.shape/stroke-width number?)
+(s/def :internal.shape/stroke-width ::safe-integer)
 (s/def :internal.shape/stroke-alignment #{:center :inner :outer})
 (s/def :internal.shape/text-align #{"left" "right" "center" "justify"})
-(s/def :internal.shape/x number?)
-(s/def :internal.shape/y number?)
-(s/def :internal.shape/cx number?)
-(s/def :internal.shape/cy number?)
-(s/def :internal.shape/width number?)
-(s/def :internal.shape/height number?)
+(s/def :internal.shape/x ::safe-integer)
+(s/def :internal.shape/y ::safe-integer)
+(s/def :internal.shape/cx ::safe-integer)
+(s/def :internal.shape/cy ::safe-integer)
+(s/def :internal.shape/width ::safe-integer)
+(s/def :internal.shape/height ::safe-integer)
 (s/def :internal.shape/index integer?)

-(s/def :internal.shape/x1 number?)
-(s/def :internal.shape/y1 number?)
-(s/def :internal.shape/x2 number?)
-(s/def :internal.shape/y2 number?)
+(s/def :internal.shape/x1 ::safe-integer)
+(s/def :internal.shape/y1 ::safe-integer)
+(s/def :internal.shape/x2 ::safe-integer)
+(s/def :internal.shape/y2 ::safe-integer)

 (s/def :internal.shape.export/suffix string?)
-(s/def :internal.shape.export/scale number?)
+(s/def :internal.shape.export/scale ::safe-integer)
 (s/def :internal.shape/export
   (s/keys :req-un [::type
                    :internal.shape.export/suffix


Fix condition in native-path calculation. Thanks to wburke.

--- src/leiningen/compile.clj
+++ src/leiningen/compile.clj
@@ -184,7 +184,7 @@
       (when native-path
         (add-system-property
          java "java.library.path" (cond
-                                   (= file (class native-path))
+                                   (instance? java.io.File native-path)
                                    (.getAbsolutePath native-path)
                                    (fn? native-path) (native-path)
                                    :default native-path)))


Handle missing .class files better in leiningen.compile/stale-namespaces.

--- src/leiningen/compile.clj
+++ src/leiningen/compile.clj
@@ -33,10 +33,12 @@
   [project]
   (filter
    (fn [n]
-     (let [clj-path (ns->path n)]
-       (> (.lastModified (file (:source-path project) clj-path))
-          (.lastModified (file (:compile-path project)
-                               (.replace clj-path "\\.clj" "__init.class"))))))
+     (let [clj-path (ns->path n)
+           class-file (file (:compile-path project)
+                            (.replace clj-path "\\.clj" "__init.class"))]
+       (or (not (.exists class-file))
+           (> (.lastModified (file (:source-path project) clj-path))
+              (.lastModified class-file)))))
    (compilable-namespaces project)))

 (defn get-by-pattern


WIP hierarchy rendering - breadcrumb naviagion across top of content implemented - Potential children still not rendered in vertical navigation on right

--- src/lonelyplanet/view.clj
+++ src/lonelyplanet/view.clj
@@ -39,7 +39,13 @@

 (deftemplate destination blank [destination taxonomy]
              [:div#header :h1] (content (get-in destination [:meta :placename]))
-             [:div.secondary-navigation :a] (content (get-in destination [:meta :placename]))
+
+             ;; breadcrumb navigation
+             [:div.secondary-navigation :ul] (content (map #(gen-hierarchy-step % taxonomy)
+                                                       (get-in destination [:meta :route])))
+             [:div.secondary-navigation [:li first-of-type]] (add-class "first")
+
+
              [:div#sidebar :div.inner] (content (html [:ul (map
                                                              #(gen-hierarchy-step % taxonomy)
                                                              (get-in destination [:meta :route]))]))


Fix some typehints

Fix some typehints

--- src/str/core.clj
+++ src/str/core.clj
@@ -148,7 +148,7 @@

 (defn- gen-padding
   "Generate the necessary padding to fill s upto width."
-  [^CharSequence s ^CharSequence padding ^Long width]
+  [^CharSequence s ^CharSequence padding ^long width]
   (let [missing (- width (.length s))
         full-lengths (Math/floor (/ missing (.length padding)))
         remaining (if (zero? full-lengths) (- width (.length s))
@@ -164,7 +164,7 @@
   width."
   ([^CharSequence s ^long width]
    (pad-right s width " "))
-  ([^CharSequence s ^Long width ^CharSequence padding]
+  ([^CharSequence s ^long width ^CharSequence padding]
    {:pre [(not-empty padding)
           (not (nil? s))]
     :post [(= (.length %) width)]}
@@ -176,9 +176,9 @@
 (defn ^String pad-left
   "Pad the beginning of s with padding, or spaces, until the length of
   s matches width."
-  ([^CharSequence s ^Long width]
+  ([^CharSequence s ^long width]
    (pad-left s width " "))
-  ([^CharSequence s ^Long width ^CharSequence padding]
+  ([^CharSequence s ^long width ^CharSequence padding]
    {:pre [(not-empty padding)
           (not (nil? s))]
     :post [(= (.length %) width)]}
@@ -190,9 +190,9 @@
 (defn ^String center
   "Pad both ends of s with padding, or spaces, until the length of s
   matches width."
-  ([^CharSequence s ^Long width]
+  ([^CharSequence s ^long width]
    (center s width " "))
-  ([^CharSequence s ^Long width ^CharSequence padding]
+  ([^CharSequence s ^long width ^CharSequence padding]
    {:pre [(not-empty padding)
           (not (nil? s))]
     :post [(= (.length %) width)]}
@@ -559,7 +559,7 @@
    :post [(string? %)]}
   (.replaceAll (.toString s) "[ \t\n\r]+" " "))

-(defn- ^Long levenshtein-distance
+(defn- levenshtein-distance
   [s1 s2]
   (let [subsolutions (atom {})
         subsolution (fn [i j]
@@ -578,7 +578,7 @@
                          1))))))
     (subsolution (.length s1) (.length s2))))

-(defn- ^Long hamming-distance [s1 s2]
+(defn- hamming-distance [s1 s2]
   (+
    (reduce + (map #(if (= %1 %2) 0 1) s1 s2))
    (- (max (.length s1) (.length s2))


Fix response format

Fix response format

--- src/antlion_clojure/bot.clj
+++ src/antlion_clojure/bot.clj
@@ -67,14 +67,18 @@
       (map->Payload {:type :message
                      :user user
                      :channel channel
-                     :text (str "ﾄｳﾛｸｼﾀﾖ!\n----------\n"
-                                answer)}))
-    (catch Exception e
-      (map->Payload {:type :message
-                     :user user
-                     :channel channel
-                     :text (str "ｺﾀｴｶﾞｵｶｼｲｰﾈ!\n----------\n"
-                                (.getMessage e))}))))
+                     :text (str "ﾄｳﾛｸｼﾀﾖ!\n"
+                                "```\n"
+                                answer
+                                "```")}))
+    (catch Exception e
+      (map->Payload {:type :message
+                     :user user
+                     :channel channel
+                     :text (str "ｺﾀｴｶﾞｵｶｼｲｰﾈ!\n"
+                                "```\n"
+                                (.getMessage e)
+                                "```")}))))

 (defn del-problem
   [{:keys [user user_profile channel] :as res} question]
@@ -103,14 +107,18 @@
       (map->Payload {:type :message
                      :user user
                      :channel channel
-                     :text (str "ﾄｳﾛｸｼﾀﾖ!\n----------\n"
-                                title ": " information)}))
-    (catch Exception e
-      (map->Payload {:type :message
-                     :user user
-                     :channel channel
-                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n----------\n"
-                                title ": " (.getMessage e))}))))
+                     :text (str "ﾄｳﾛｸｼﾀﾖ!\n"
+                                "```\n"
+                                title ": " information
+                                "```")}))
+    (catch Exception e
+      (map->Payload {:type :message
+                     :user user
+                     :channel channel
+                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n"
+                                "```\n"
+                                title ": " (.getMessage e)
+                                "```")}))))

 (defn- del-fyi
   [{:keys [user user_profile channel] :as res} title]
@@ -120,19 +128,25 @@
         (map->Payload {:type :message
                        :user user
                        :channel channel
-                       :text (str "ﾐﾂｶﾗﾅｲ−ﾖ!\n----------\n"
-                                  title)})
-        (map->Payload {:type :message
-                       :user user
-                       :channel channel
-                       :text (str "ｹｼﾀﾖ!\n----------\n"
-                                  title)})))
-    (catch Exception e
-      (map->Payload {:type :message
-                     :user user
-                     :channel channel
-                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n----------\n"
-                                title ": " (.getMessage e))}))))
+                       :text (str "ﾐﾂｶﾗﾅｲ−ﾖ!\n"
+                                  "```\n"
+                                  title
+                                  "```")})
+        (map->Payload {:type :message
+                       :user user
+                       :channel channel
+                       :text (str "ｹｼﾀﾖ!\n"
+                                  "```\n"
+                                  title
+                                  "```")})))
+    (catch Exception e
+      (map->Payload {:type :message
+                     :user user
+                     :channel channel
+                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n"
+                                "```\n"
+                                title ": " (.getMessage e)
+                                "```")}))))

 (defn- format-fyi
   [fyis]
@@ -162,8 +176,10 @@
       (map->Payload {:type :message
                      :user user
                      :channel channel
-                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n----------\n"
-                                title ": " (.getMessage e))}))))
+                     :text (str "ﾊﾞｶｼﾞｬﾈｰﾉ\n"
+                                "```\n"
+                                title ": " (.getMessage e)
+                                "```")}))))

 (defn- eval
   [{:keys [user channel] :as res} txt]
@@ -229,7 +245,6 @@
     :as res}]
   (let [self (redis/get-self)
         txt (-> (split text #"\s+" 2) second)]
-    (println txt)
     (when (slack/message-for-me? res self)
       (cond
         (redis/get-checking-question? user)


fix typo

fix typo

--- src/metabase/api/card.clj
+++ src/metabase/api/card.clj
@@ -21,7 +21,7 @@
 (defendpoint GET "/"
   "Get all the `Cards` for an `Org`. With param `f` (default is `all`), restrict cards as follows:

-   *  `all`  Return all `Cards` for `Org` which were create by current user or are publicly visible
+   *  `all`  Return all `Cards` for `Org` which were created by current user or are publicly visible
    *  `mine` Return all `Cards` for `Org` created by current user
    *  `fav`  Return all `Cards` favorited by the current user"
   [org f]


Use str instead of s alias for clojure.string.

--- src/mvxcvi/crypto/pgp.clj
+++ src/mvxcvi/crypto/pgp.clj
@@ -3,7 +3,7 @@
   (:require
     byte-streams
     [clojure.java.io :as io]
-    [clojure.string :as string]
+    [clojure.string :as str]
     [mvxcvi.crypto.util :refer [do-bytes hex-str]])
   (:import
     (java.io
@@ -30,7 +30,7 @@
       BcPBESecretKeyDecryptorBuilder)))

-;; CONFIGURATION
+;; CONSTANTS / CONFIGURATION

 (defn- map-tags
   "Converts static 'tag' fields on the given class into a map of keywords to
@@ -39,7 +39,7 @@
   (let [field->entry
         (fn [^java.lang.reflect.Field f]
           (vector (-> (.getName f)
-                      (string/replace \_ \-)
+                      (str/replace \_ \-)
                       .toLowerCase
                       keyword)
                   (.getInt f nil)))]
@@ -148,7 +148,7 @@
          :algorithm (key-algorithm pubkey)
          :fingerprint (->> (.getFingerprint pubkey)
                            (map (partial format "%02X"))
-                           string/join)
+                           str/join)
          :encryption-key? (.isEncryptionKey pubkey)
          :user-ids (-> pubkey .getUserIDs iterator-seq vec)}]
     (if (instance? PGPSecretKey k)


Additional sanity check for whether the lifecycles were installed

--- src/onyx/plugin/kafka.clj
+++ src/onyx/plugin/kafka.clj
@@ -251,10 +251,11 @@
   (read-batch [_ event]
     (let [pending (count @pending-messages)
           max-segments (max (min (- max-pending pending) batch-size) 0)
+          consumer (or (:kafka/consumer event) (throw (Exception. "Kafka consumer not found in event map. Did you include the Kafka input lifecycles?")))
           _ (when (and (not (zero? max-segments))
                        (or (nil? @iter)
                            (not (.hasNext ^java.util.Iterator @iter))))
-              (reset! iter (.iterator ^ConsumerRecords (.poll ^Consumer (.consumer ^FranzConsumer (:kafka/consumer event)) batch-timeout))))
+              (reset! iter (.iterator ^ConsumerRecords (.poll ^Consumer (.consumer ^FranzConsumer consumer) batch-timeout))))
           batch (transient [])
           n-retries (take-values! batch retry-ch batch-size)
           _ (take-records! batch @iter segment-fn (- max-segments n-retries))


use consistent prefix; use superstring for utility fn

--- ssclj/jar/src/main/clojure/com/sixsq/slipstream/ssclj/resources/common/utils.clj
+++ ssclj/jar/src/main/clojure/com/sixsq/slipstream/ssclj/resources/common/utils.clj
@@ -3,10 +3,10 @@
   (:require
     [clojure.tools.logging                                    :as log]
     [clojure.edn                                              :as edn]
-    [superstring.core                                           :refer [split]]
+    [superstring.core                                         :as s]
     [clj-time.core                                            :as time]
     [clj-time.format                                          :as time-fmt]
-    [schema.core                                              :as s]
+    [schema.core                                              :as schema]
     [ring.util.response                                       :as r]
     [com.sixsq.slipstream.ssclj.resources.common.debug-utils  :as du]
     [clojure.data.json                                        :as json])
@@ -20,8 +20,10 @@
 ;; embedded in them
 ;;

+;; NOTE: this cannot be replaced with s/lisp-case because it
+;; will treat a '/' in a resource name as a word separator.
 (defn de-camelcase [str]
-  (superstring.core/join "-" (map superstring.core/lower-case (superstring.core/split str #"(?=[A-Z])"))))
+  (s/join "-" (map s/lower-case (s/split str #"(?=[A-Z])"))))

 (defn json-response
   [body]
@@ -77,7 +79,7 @@
 (defn resource-name
   [resource-id]
   (-> resource-id
-      (split #"/")
+      (s/split #"/")
       first))

 ;;
@@ -121,7 +123,7 @@
    violations of the schema and a 400 ring response. If everything's
    OK, then the resource itself is returned."
    [schema]
-   (let [checker (s/checker schema)]
+   (let [checker (schema/checker schema)]
     (fn [resource]
       (if-let [msg (checker resource)]
         (let [msg (str "resource does not satisfy defined schema: " msg)
@@ -191,9 +193,7 @@
     (do
       (log/warn s " is not lisp-cased.")
       "")
-    (->>  (superstring.core/split s #"-")
-          (map superstring.core/capitalize)
-          (apply str))))
+    (s/pascal-case s)))

 (defn map-multi-line
   [m]


Slightly nicer js*

Slightly nicer js*

--- src/aurora/interpreter.cljs
+++ src/aurora/interpreter.cljs
@@ -182,7 +182,7 @@
    (println (.join (make-array indent) " ") "=>" (.-id frame) (.-vars frame))
    (doseq [call (.-calls frame)]
      (print-stack (+ indent 2) call))
-   (when (js* "(~{} in ~{})" "result" frame) ; seriously?
+   (when (js* "('result' in ~{frame})") ; seriously?
      (println (.join (make-array indent) " ") "<=" (.-id frame) (.-result frame)))))

 (defn print-example [example inputs]
@@ -190,5 +190,7 @@
     (print-stack stack)
     (println result)))

-(run-example example-b [1 "foo"])
+(print-example example-b [[1 "foo"]])
+(print-example example-b [1 "foo"])
+(print-example example-c [10])


moved all implications to encyc

moved all implications to encyc

--- src/babel/encyclopedia.cljc
+++ src/babel/encyclopedia.cljc
@@ -44,25 +44,59 @@
                        :buyable true
                        :furniture false
                        :legible false
+                       :human false
+                       :part-of-human-body false
                        :pet false
                        :physical-object true
                        :speakable false}

    {:consumable false} {:drinkable false
                         :edible false}
-
-   {:human true} {:animate true}

+   {:drinkable true}   {:mass true}
+
+   {:edible true}      {:consumable true}
+
+   {:furniture true}   {:artifact true
+                        :animate false
+                        :buyable true
+                        :consumable false
+                        :legible false
+                        :edible false
+                        :place false
+                        :speakable false}
+
+   {:human true} {:animate true
+                  :buyable false
+                  :consumable false
+                  :legible false
+                  :pet false
+                  :part-of-human-body true
+                  :physical-object true
+                  :place false}
+
+   {:living false} {:animate false
+                    :human false}
+
    {:living true} {:artifact false}

+   {:part-of-human-body true} {:consumable false
+                               :human false
+                               :physical-object true}
+
    {:pet true} {:animate true
                 :buyable true
                 :edible false
                 :human false}

-   {:pred :house} {:artifact true
-                   :buyable true
-                   :place true}
+   {:place true} {:activity false
+                  :consumable false
+                  :living false
+                  :physical-object true}
+
+   {:time true} {:activity false
+                 :living false
+                 :place false}

    }
   )
@@ -95,138 +129,15 @@
             city       (get-encyc input :city)
             clothing   (get-encyc input :clothing)
             consumable (get-encyc input :consumable)
-
-            drinkable
-            ;; drinkables are always mass nouns.
-            (if (= (get-in input '(:drinkable)) true)
-              {:mass true})
-
-            drinkable-xor-edible-1
-            ;; things are either drinkable or edible, but not both (except for weird foods
-            ;; like pudding or soup). (part 1: edible)
-            (if (and (= (get-in input '(:edible)) true)
-                     (= (get-in input '(:drinkable) :notfound) :notfound))
-              {:drinkable false}{})
-
-            drinkable-xor-edible-2
-            ;; things are either drinkable or edible, but not both (except for weird foods
-            ;; like pudding or soup). (part 2: drinkable)
-            (if (and (= (get-in input '(:drinkable)) true)
-                     (= (get-in input '(:edible) :notfound) :notfound))
-              {:edible false})
-
-            ;; qualities of foods and drinks.
-            edible (if (or (= (get-in input '(:edible)) true)
-                           (= (get-in input '(:drinkable)) true))
-                     {:consumable true
-                      :human false
-                      :pet false
-                      :place false
-                      :speakable false
-                      :legible false
-                      :furniture false
-                      :part-of-human-body false}{})
-
-            furniture (if (= (get-in input '(:furniture))
-                             true)
-                        {:artifact true
-                         :animate false
-                         :buyable true
-                         :drinkable false
-                         :legible false
-                         :edible false
-                         :place false
-                         :speakable false})
-
-            human (if (= (get-in input '(:human))
-                         true)
-                    {:activity false
-                     :buyable false
-                     :physical-object true
-                     :edible false
-                     :animate true
-                     :part-of-human-body false
-                     :drinkable false
-                     :speakable false
-                     :city false
-                     :place false})
-            inanimate (if (= (get-in input '(:animate))
-                             false)
-                        {:human false})
-
-            ;; legible(x) => artifact(x),drinkable(x,false),edible(x,false),human(x,false)
-            legible
-            (if (= (get-in input '(:legible)) true)
-              {:artifact true
-               :drinkable false
-               :human false
-               :furniture false
-               :part-of-human-body false
-               :edible false})
-
-            material-false
-            (if (= (get-in input '(:material)) :false)
-              {:edible false
-               :animate false
-               :drinkable false
-               :buyable false ; money can't buy me love..
-               :visible false})
-
-            non-places (if (or
-                            (= (get-in input '(:legible)) true)
-                            (= (get-in input '(:part-of-human-body)) true)
-                            (= (get-in input '(:pred)) :fiore)
-                            (= (get-in input '(:pred)) :scala))
-                         {:place false})
-
-            ;; artifact(x,false) => legible(x,false)
-            not-legible-if-not-artifact
-            (if (= (get-in input '(:artifact)) false)
-              {:legible false})
-
-            part-of-human-body
-            (if (= (get-in input '(:part-of-human-body)) true)
-              {:speakable false
-               :buyable false
-               :animate false
-               :edible false
-               :drinkable false
-               :legible false
-               :artifact false})
-
-            ;; we don't eat pets (unless things get so desperate that they aren't pets anymore)
-            pets (if (= (get-in input '(:pet))
-                        true)
-                   {:edible false
-                    :buyable true
-                    :physical-object true
-                    })
-
-            place (if (= (get-in input '(:place))
-                         true)
-                    {:activity false
-                     :animate false
-                     :speakable false
-                     :physical-object true
-                     :drinkable false
-                     :edible false
-                     :legible false}{})
-
-            place-false (if (= (get-in input '(:place))
-                               false)
-                          {:city false})
-
-            time (if (= (get-in input [:time])
-                        true)
-                   {:activity false
-                    :animate false
-                    :city false
-                    :speakable false
-                    :physical-object true
-                    :drinkable false
-                    :edible false
-                    :legible false
-                    :place false})
+            drinkable  (get-encyc input :drinkable)
+            edible     (get-encyc input :edible)
+            furniture  (get-encyc input :furniture)
+            human      (get-encyc input :human)
+            part-of-human-body (get-encyc input
+                                          :part-of-human-body)
+            pet        (get-encyc input :pet)
+            place      (get-encyc input :place)
+            time       (get-encyc input :time)
             ]
         (let [merged
               (cond (= input :fail) :fail
@@ -234,12 +145,8 @@
                     true
                     (merge input
                            activity animate artifact buyable city clothing consumable drinkable
-                           drinkable-xor-edible-1 drinkable-xor-edible-2
-                           edible furniture human inanimate
-                           legible material-false non-places
-                           not-legible-if-not-artifact part-of-human-body pets place
-                           place-false time
-                           ))]
+                           edible furniture human
+                           pet place time))]
           (log/trace (str "sem-impl so far: " merged))
           (if (not (= merged input)) ;; TODO: make this check more efficient: count how many rules were hit
             ;; rather than equality-check to see if merged has changed.


fixed parameter typo in state methods

fixed parameter typo in state methods

--- src/braid/common/state.cljs
+++ src/braid/common/state.cljs
@@ -113,11 +113,11 @@
       (filter (fn [tag] @(get-user-subscribed-to-tag @state [nil (tag :id)]))))))

 (defn get-user-avatar-url
-  [state user_id]
+  [state user-id]
   (reaction (get-in @state [:users user-id :avatar])))

 (defn get-user-status
-  [state user_id]
+  [state user-id]
   (reaction (get-in @state [:users user-id :status])))

 (defn get-search-query


Using single storage stom.

Using single storage stom.

--- src/cbfg/world-storage.cljs
+++ src/cbfg/world-storage.cljs
@@ -6,36 +6,45 @@
                               get-el-innerHTML set-el-innerHTML]]
             [cbfg.fence :refer [make-fenced-pump]]))

-(defn make-storage-cmd-handlers [kvs changes last-seq max-deleted-seq]
+(defn storage-get [storage opaque-id key]
+  (let [s @storage
+        sq (get (:kvs s) key)
+        change (when sq (get (:changes s) sq))]
+    (if (and change (not (:deletion change)))
+      [:ok (:val change)]
+      [:not-found nil])))
+
+(defn make-storage-cmd-handlers [storage]
   {"get" [["key"]
           (fn [c] {:rq (fn [actx]
-                         (ago storage-get actx
-                              {:opaque-id (:opaque-id c) :status :ok
-                               :key (:key c) :val "yay"}))})]
+                         (ago storage-cmd-get actx
+                              (let [[status val] (storage-get storage (:opaque-id c) (:key c))]
+                                {:opaque-id (:opaque-id c) :status status
+                                 :key (:key c) :val val})))})]
    "set" [["key" "val"]
           (fn [c] {:rq (fn [actx]
-                         (ago storage-get actx
+                         (ago storage-cmd-set actx
                               {:opaque-id (:opaque-id c) :status :ok
                                :key (:key c)}))})]
    "del" [["key"]
           (fn [c] {:rq (fn [actx]
-                         (ago storage-get actx
+                         (ago storage-cmd-del actx
                               {:opaque-id (:opaque-id c) :status :ok
                                :key (:key c)}))})]
    "scan" [["from" "to"]
            (fn [c] {:rq (fn [actx]
-                          (ago storage-get actx
+                          (ago storage-cmd-scan actx
                                {:opaque-id (:opaque-id c) :status :ok
                                 :from (:from c) :to (:to c)}))})]
    "changes" [["since"]
               (fn [c] {:rq
                        (fn [actx]
-                         (ago storage-get actx
+                         (ago storage-cmd-changes actx
                               {:opaque-id (:opaque-id c) :status :ok
                                :since (:since c)}))})]
    "noop" [[] (fn [c] {:rq
                        (fn [actx]
-                         (ago storage-get actx
+                         (ago storage-cmd-noop actx
                               {:opaque-id (:opaque-id c) :status :ok}))})]
    "test" [[] (fn [c] {:rq #(cbfg.fence/test % (:opaque-id c))})]})

@@ -47,12 +56,8 @@
                          (get-el-innerHTML (str el-prefix "-" log-kind "-log")))))

 (defn world-vis-init [el-prefix]
-  (let [kvs (atom {})
-        changes (atom {})
-        last-seq (atom 0)
-        max-deleted-seq (atom 0)
-        storage-cmd-handlers (make-storage-cmd-handlers kvs changes
-                                                        last-seq max-deleted-seq)
+  (let [storage (atom {:kvs {} :changes {} :last-sq 0 :max-deleted-sq 0})
+        storage-cmd-handlers (make-storage-cmd-handlers storage)
         cmd-ch (map< (fn [ev] {:op (.-id (.-target ev))})
                      (merge (map #(listen-el (gdom/getElement %) "click")
                                  (keys storage-cmd-handlers))))]
@@ -67,9 +72,8 @@
                                   cmd2 (-> cmd
                                            (assoc-in [:opaque-id] num-ins)
                                            (assoc-in [:fence] op-fence))
-                                  cmd3 (reduce #(assoc %1
-                                                  (symbol (str ":" %2))
-                                                  (get-el-value (str op "-" %2)))
+                                  cmd3 (reduce #(assoc %1 (keyword %2)
+                                                       (get-el-value (str op "-" %2)))
                                                cmd2 params)]
                               (el-log el-prefix "input" (str num-ins ": " cmd3))
                               (aput a-input in-ch (handler cmd3))


nicer core

nicer core

--- src/selectscript/core.clj
+++ src/selectscript/core.clj
@@ -37,8 +37,8 @@
    ["-p" "--parse-tree"  "print parsetree"              :default false  :assoc-fn (fn [m k _] (update-in m [k] not))]
    ["-x" "--execute"     "Run the program"              :default false  :assoc-fn (fn [m k _] (update-in m [k] not))]
    ["-r" "--repl"        "interactive"                  :default false  :assoc-fn (fn [m k _] (update-in m [k] not))]
-   ["-s" "--server"     "run compiler as web-server"    :default false  :assoc-fn (fn [m k _] (update-in m [k] not))]
-   [nil  "--port"       "server port"                   :default 8080   :parse-fn #(Integer/parseInt %)]
+   ["-s" "--server"      "run compiler as web-server"   :default false  :assoc-fn (fn [m k _] (update-in m [k] not))]
+   [nil  "--port"        "server port"                  :default 8080   :parse-fn #(Integer/parseInt %)]
    ;; A boolean option defaulting to nil
    ["-h" "--help"]])


Improve test-var and test-ns, accept TestCases in :test metadata

--- lazytest.clj
+++ lazytest.clj
@@ -272,24 +272,52 @@
 ;;; TESTABLE IMPLEMENTATIONS

 (defn var-test-case
-  "Creates a TestCase for a Var, using clojure.core/test."
-  [v]
-   #^{:name (:name (meta v))}
-   (TestCase [] [(assertion [] (clojure.core/test v))]))
+  "Returns a TestCase for a Var.  If the Var's value is a TestCase,
+  returns that.  If the Var's :test metadata is a TestCase, returns
+  that.  If the Var's :test metadata is a function, generates a
+  TestCase with an assertion that calls the function.  If none of
+  these are true, returns nil."  [v]
+  (cond (= ::TestCase (type (var-get v)))
+        (var-get v)
+
+        (= ::TestCase (type (:test (meta v))))
+        (:test (meta v))
+
+        (fn? (:test (meta v)))
+        (with-meta (TestCase [] [(assertion [] ((:test (meta v))))])
+          {:doc "Var :test metadata function."
+           :name (:name (meta v))
+           :ns (:ns (meta v))})
+
+        :else nil))

 (defn test-var
-  "Creates and runs a TestCase for the Var."
+  "Finds and runs a TestCase for the Var."
   [v]
   (run-test-case (var-test-case v)))

 (defn ns-test-case
-  "Creates a TestCase for the namespace that tests all its Vars."
-  [n]
-  #^{:name (ns-name n)}
-  (TestCase [] (map var-test-case (ns-interns n))))
+  "Returns a TestCase for a Namespace.  If the namespace's :test
+  metadata is a TestCase, returns that.  If the namespace's :test
+  metadata is another namespace, recurses on that.  If the
+  namespace's :test metadata is a function, generates a TestCase with
+  an assertion that calls the function.  If none of these are true,
+  returns nil."  [n]
+  (cond (= ::TestCase (type (:test (meta n))))
+        (:test (meta n))
+
+        (instance? clojure.lang.Namespace (:test (meta n)))
+        (ns-test-case (:test (meta n)))
+
+        (fn? (:test (meta n)))
+        (with-meta (TestCase [] [(assertion [] ((:test (meta n))))])
+          {:doc "Namespace :test metadata function."
+           :name (ns-name n)})
+
+        :else nil))

 (defn test-ns
-  "Creates and runs a TestCase for the namespace."
+  "Finds and runs a TestCase for the namespace."
   [n]
   (run-test-case (ns-test-case n)))


increase max-total-depth from 5 to 6

increase max-total-depth from 5 to 6

--- src/babel/generate.cljc
+++ src/babel/generate.cljc
@@ -9,7 +9,7 @@

 ;; during generation, will not decend deeper than this when creating a tree:
 ;; TODO: should also be possible to override per-language.
-(def ^:const max-total-depth 5)
+(def ^:const max-total-depth 6)
 (def ^:const max-generated-complements 200)

 ;; use map or pmap.


+(defn create-mapping)

+(defn create-mapping)

--- src/babel/generate.cljc
+++ src/babel/generate.cljc
@@ -50,7 +50,14 @@
                         (flatten (vals
                                   (or (:lexicon (:generate model)) (:lexicon model)))))))
     []))
-
+
+(defn create-mapping [bolt comp-paths model depth max-depth]
+  (zipmap comp-paths
+          (map (fn [path]
+                 (babel.generate/comp-path-to-bolts
+                  bolt path model depth max-depth))
+               comp-paths)))
+
 (defn comp-path-to-bolts
   "return a lazy sequence of bolts for all possible complements that can be added to the end of the _path_ within _bolt_."
   [bolt path model depth max-depth]


+debug logging

+debug logging

--- src/babel/generate.cljc
+++ src/babel/generate.cljc
@@ -170,6 +170,10 @@
                            (filter #(= true
                                        (get-in % [:head :phrasal] true))
                                    parents)))]
+      (log/debug (str "lexical-heads for parents:" (string/join "," (map :rule parents)) ":"
+                      (string/join ","
+                                   (map #((:morph language-model) %)
+                                        lexical))))
       (if (lexemes-before-phrases total-depth max-total-depth)
         (lazy-cat lexical phrasal)
         (lazy-cat phrasal lexical)))))


more small speedups of dubious comprehensibility

--- src/babel/generate.cljc
+++ src/babel/generate.cljc
@@ -110,8 +110,8 @@
                                                              (+ 1 depth) index morph (+ 1 total-depth))))
                         parents))]
           (if (lexemes-before-phrases total-depth)
-            (lazy-cat lexical phrasal)
-            (lazy-cat phrasal lexical)))))))
+            (concat lexical phrasal)
+            (concat phrasal lexical)))))))

 (defn add-complement [bolt path spec grammar lexicon cache morph depth total-depth]
   (log/info (str "add-complement: " (show-bolt bolt path morph)))
@@ -134,8 +134,7 @@
                                                     (get-in child-in-bolt [:synsem] :top)))))))
         filtered-lexical-complements (filter (fn [lexeme]
                                                (complement-pre-check lexeme bolt path))
-                                             complement-candidate-lexemes)
-        shuffled-candidate-lexical-complements (lazy-shuffle filtered-lexical-complements)]
+                                             complement-candidate-lexemes)]
     (filter (fn [complement]
               (if (fail? complement)
                 (do
@@ -158,7 +157,7 @@
                                               "): calling generate-all(" (strip-refs spec) ");"
                                               "input-spec: " input-spec))
                         phrasal-complements (if (> max-total-depth total-depth)
-                                              (generate-all spec grammar lexicon cache morph (+ depth total-depth)))]
+                                              (lazy-seq (generate-all spec grammar lexicon cache morph (+ depth total-depth))))]
                     (if (lexemes-before-phrases total-depth)
                       (lazy-cat filtered-lexical-complements phrasal-complements)
                       (lazy-cat phrasal-complements filtered-lexical-complements)))))))


Remove duplicate log

Remove duplicate log

--- src/buildviz/go/api.clj
+++ src/buildviz/go/api.clj
@@ -28,7 +28,7 @@
                   true))

-;; /properties/%pipeline/%pipeline_run/%stage/%stage_run/%job
+;; /api/jobs/%d.xml

 (defn- handle-missing-start-time-when-cancelled [build-start-time build-end-time]
   (if (nil? build-start-time)
@@ -160,6 +160,5 @@
          (mapcat filter-xml-files file-tree))))

 (defn get-junit-xml [go-url job-instance]
-  (when-let [xml-file-urls (xml-artifacts-for-job-run go-url job-instance)]
-    (log/info (format "Reading test results from %s" xml-file-urls))
+  (when-let [xml-file-urls (seq (xml-artifacts-for-job-run go-url job-instance))]
     (map #(get-plain go-url %) xml-file-urls)))


Spawning client from world for vis displayability.

--- src/cbfg/world-net.cljs
+++ src/cbfg/world-net.cljs
@@ -44,11 +44,11 @@
            (aput client-loop connect-ch [server-addr server-port client-addr connect-result-ch])
            (when-let [[client-send-ch client-recv-ch close-client-recv-ch]
                       (atake client-loop connect-result-ch)]
-             (ago-loop client-loop-in client-loop [num-ins 0]
+             (ago-loop client-loop-in actx [num-ins 0]
                        (when-let [msg (atake client-loop-in cmd-ch)]
                          (aput client-loop-in client-send-ch [msg])
                          (recur (inc num-ins))))
-             (ago-loop client-loop-out client-loop [num-outs 0]
+             (ago-loop client-loop-out actx [num-outs 0]
                        (when-let [msg (atake client-loop-out client-recv-ch)]
                          (aput client-loop-out res-ch msg)
                          (recur (inc num-outs)))))))
@@ -208,11 +208,11 @@
                          (server-accept-loop world accept-ch1 close-accept-ch1)
                          (let [req-ch (achan init-world)
                                res-ch (achan init-world)
-                               client-cmd-chs {"client-0" (client-loop init-world connect-ch
+                               client-cmd-chs {"client-0" (client-loop world connect-ch
                                                                        :server 8000 :client-0 res-ch)
-                                               "client-1" (client-loop init-world connect-ch
+                                               "client-1" (client-loop world connect-ch
                                                                        :server 8100 :client-1 res-ch)
-                                               "client-2" (client-loop init-world connect-ch
+                                               "client-2" (client-loop world connect-ch
                                                                        :server 8000 :client-2 res-ch)}]
                            (world-cmd-loop init-world cbfg.world-lane/cmd-handlers cmd-ch
                                            req-ch res-ch vis-chs world-vis-init el-prefix)


remove incorrect destructuring

remove incorrect destructuring

--- src/clj/cljs/source_map.clj
+++ src/clj/cljs/source_map.clj
@@ -120,7 +120,7 @@
 (defn encode [m opts]
   (let [names->idx (atom {})
         name-idx   (atom 0)
-        {:keys [segs]}
+        segs
         (loop [sources (seq m) prev-seg nil source-idx 0 segs []]
           (if sources
             (let [[source lines] (first sources)


Append a \newline at the end of the header string

--- src/cljam/io/bcf/writer.clj
+++ src/cljam/io/bcf/writer.clj
@@ -47,6 +47,7 @@
                     (concat
                      (stringify-meta (.meta-info w))
                      [(vw/stringify-header (.header w))]))
+                   (str \newline) ;; newline at the end of the header
                    (str (char 0)) ;; NULL-terminated
                    .getBytes)
         hlen (alength hdr-ba)]