Improved reddit video requests query to be more reliable

--- src/cljs/redditv/reddit/request.cljs
+++ src/cljs/redditv/reddit/request.cljs
@@ -126,10 +126,15 @@
   offset. [default: nil]

   vcount - provide an offset on what is requested from a particular
-  subreddit
+  subreddit [default: 0]

   allow-nsfw? - If false, will filter out videos containing
   Not-Safe-For-Work content [default: true]
+
+  max-requests - Maximum number of requests to perform to get the
+  desired number of video posts. Upon reaching the max-requests, it
+  will return video posts it was able to scrape in the resulting
+  number of requests.

   Return Value:

@@ -145,21 +150,19 @@
         max-requests 15]

     (go-loop [video-posts '()
-              num-requests 0]
+              num-requests 0
+              after-id after]

       ;; Query the reddit api until we've reached the desired number
       ;; of video posts, or until we've reached the max number of
       ;; request we're willing to make
       (if (and (< (count video-posts) rlimit)
                (< num-requests max-requests))
-        (let [limit (min (- rlimit (count video-posts)) 100)
-              vcount (count video-posts)
-              after-id (or (-> video-posts last :name) after)
-
+        (let [
               ;; Generate the request and query reddit using jsonp
               url (generate-subreddit-request-url
-                   {:subreddit subreddit :limit limit :category category
-                    :after after-id :vcount vcount})
+                   {:subreddit subreddit :limit 100 :category category
+                    :after after-id :vcount (count video-posts)})
               [s-chan e-chan] (send-jsonp url)

               ;; Convert the resulting JS Object into a clojure collection of map objects
@@ -167,16 +170,18 @@
               result (map #(:data %) data-result)

               ;; Remove NSFW Posts if flag is not set
-              result (if-not allow-nsfw? (filter (complement post-is-nsfw?) result) result)
+              clean-result (if-not allow-nsfw? (filter (complement post-is-nsfw?) result) result)

               ;; Scrape out any posts that don't contain playable videos
-              video-results (filter post-is-video? result)
+              video-results (filter post-is-video? clean-result)

               ;; Remove any distinct reddit posts from subsequent reddit api requests
               video-results (distinct-by-id video-results)
               ]
-          (recur (concat video-posts video-results)
-                 (inc num-requests))))
+          (recur (take rlimit (concat video-posts video-results)) ; video-posts
+                 (inc num-requests) ; num-requests
+                 (or (-> result last :name) after-id)) ; after-id
+          ))
       (>! main-data-channel video-posts))
     [main-data-channel main-error-channel]))


Fix copyright year on shmac ns.

Fix copyright year on shmac ns.

--- src/clojure/buddy/core/mac/shmac.clj
+++ src/clojure/buddy/core/mac/shmac.clj
@@ -1,4 +1,4 @@
-;; Copyright 2014 Andrey Antukh <niwi@niwi.be>
+;; Copyright 2014-2015 Andrey Antukh <niwi@niwi.be>
 ;;
 ;; Licensed under the Apache License, Version 2.0 (the "License")
 ;; you may not use this file except in compliance with the License.


Fix null pointer exception on loads signing function.

--- src/clojure/buddy/crypto/signing.clj
+++ src/clojure/buddy/crypto/signing.clj
@@ -49,4 +49,5 @@
   "Unsign data signed with dumps."
   [s & args]
   (let [unsigned (apply unsign s (vec args))]
-    (nippy/thaw (base64->bytes unsigned))))
+    (when-not (nil? unsigned)
+      (nippy/thaw (base64->bytes unsigned)))))


CMR-339: Added lower case indexing of project-refs.

--- src/cmr/indexer/services/granule.clj
+++ src/cmr/indexer/services/granule.clj
@@ -80,6 +80,7 @@
      :granule-ur granule-ur
      :granule-ur.lowercase (s/lower-case granule-ur)
      :project-refs project-refs
+     :project-refs.lowercase (map s/lower-case project-refs)
      :attributes (psa-refs->elastic-docs parent-collection umm-granule)
      :start-date (when start-date (f/unparse (f/formatters :date-time) start-date))
      :end-date (when end-date (f/unparse (f/formatters :date-time) end-date))}))


use string instead of symbol for :name

use string instead of symbol for :name

--- src/com/billpiel/mem_tracer/core.clj
+++ src/com/billpiel/mem_tracer/core.clj
@@ -71,7 +71,7 @@
          s  (.sym v)]
      (if (and (ifn? @v) (-> v meta :macro not) (-> v meta ::traced not))
        (let [f @v
-             vname (symbol (str ns "/" s))] ;; string!!
+             vname (str ns "/" s)]
          (doto v
            (alter-var-root #(fn tracing-wrapper [& args]
                               (println "IN TRACING WRAPPER!")


Maker server load devel middleware depending on env

--- src/madouc/core.clj
+++ src/madouc/core.clj
@@ -1,5 +1,8 @@
 (ns madouc.core
-  (:require [immutant.web :as server]
+  (:require [ring.logger.timbre :as logger]
+            [immutant.web :as server]
+            [ring.middleware.reload :refer [wrap-reload]]
+            [ring.middleware.stacktrace :refer [wrap-stacktrace]]
             [madouc.handlers :as app]
             [environ.core :refer [env]]
             [taoensso.timbre :as timbre])
@@ -12,8 +15,25 @@
   (server/stop)
   (timbre/info "Server stopped"))

+(defn wrap-with-dev
+  [orig-ep]
+  (if (= (env :madouc-env "dev"))
+    (do
+      (timbre/info "Server started with *development* configuration")
+      (-> orig-ep
+          wrap-reload
+          wrap-stacktrace))
+    orig-ep))
+
+(defn make-entrypoint
+  [handler]
+  (-> handler
+      logger/wrap-with-logger
+      wrap-with-dev))
+
 (defn -main
   [& args]
   (.addShutdownHook (Runtime/getRuntime) (Thread. #(stop-server)))
   (timbre/info "Starting server...")
-  (server/run app/entrypoint))
+  (let [ep (make-entrypoint app/main-handler)]
+    (server/run ep)))


[#2091] Track metrics on flow-api/check_permissions

--- backend/src/akvo/lumen/lib/auth.clj
+++ backend/src/akvo/lumen/lib/auth.clj
@@ -3,17 +3,19 @@
    [akvo.commons.jwt :as jwt]
    [akvo.lumen.component.flow :as c.flow]
    [akvo.lumen.component.tenant-manager :as tenant-manager]
+   [akvo.lumen.monitoring :as monitoring]
+   [akvo.lumen.protocols :as p]
    [akvo.lumen.specs :as lumen.s]
+   [akvo.lumen.specs.collection :as collection.s]
+   [akvo.lumen.specs.dashboard :as dashboard.s]
    [akvo.lumen.specs.dataset :as dataset.s]
    [akvo.lumen.specs.visualisation :as visualisation.s]
-   [akvo.lumen.specs.dashboard :as dashboard.s]
-   [akvo.lumen.specs.collection :as collection.s]
-   [akvo.lumen.protocols :as p]
+   [clojure.set :as set]
    [clojure.spec.alpha :as s]
    [clojure.string :as str]
    [clojure.tools.logging :as log]
-   [clojure.set :as set]
    [hugsql.core :as hugsql]
+   [iapetos.core :as prometheus]
    [integrant.core :as ig]))

 (hugsql/def-db-fns "akvo/lumen/lib/visualisation.sql")
@@ -133,6 +135,29 @@
                                   :visualisation-ids auth-visualisations
                                   :dashboard-ids auth-dashboards})))

+(defn- flow-check-permissions [flow-api request collector tenant data]
+  (let [t (prometheus/start-timer collector :app/flow-check-permissions {"tenant" tenant})
+        res (c.flow/check-permissions flow-api (jwt/jwt-token request) data)]
+    (t)
+    res))
+
+(defn- load-auth-data [dss rasters tenant-conn flow-api request collector tenant]
+  (let [permissions         (->> (map :source dss)
+                                 (filter #(= "AKVO_FLOW" (get % "kind")))
+                                 (map c.flow/>api-model)
+                                 (flow-check-permissions flow-api request collector tenant)
+                                 :body
+                                 set)
+        auth-datasets       (auth-datasets dss permissions)
+        auth-visualisations (auth-visualisations tenant-conn (set auth-datasets))
+        auth-dashboards     (auth-dashboards tenant-conn (set auth-visualisations))
+        auth-collections    (auth-collections tenant-conn auth-datasets auth-visualisations auth-dashboards)]
+    {:rasters             rasters
+     :auth-datasets       auth-datasets
+     :auth-visualisations auth-visualisations
+     :auth-dashboards     auth-dashboards
+     :auth-collections    auth-collections}))
+
 (defn wrap-auth-datasets
   "Add to the request an auth-service protocol impl using flow-api check_permissions"
   [tenant-manager flow-api collector]
@@ -143,21 +168,7 @@
             rasters        (mapv :id (all-rasters tenant-conn))
             auth-uuid-tree (if (and (match-by-jwt-family-name? request)
                                     (match-by-template-and-method? auth-calls request))
-                             (let [permissions         (->> (map :source dss)
-                                                            (filter #(= "AKVO_FLOW" (get % "kind")))
-                                                            (map c.flow/>api-model)
-                                                            (c.flow/check-permissions flow-api (jwt/jwt-token request))
-                                                            :body
-                                                            set)
-                                   auth-datasets       (auth-datasets dss permissions)
-                                   auth-visualisations (auth-visualisations tenant-conn (set auth-datasets))
-                                   auth-dashboards     (auth-dashboards tenant-conn (set auth-visualisations))
-                                   auth-collections    (auth-collections tenant-conn auth-datasets auth-visualisations auth-dashboards)]
-                               {:rasters             rasters
-                                :auth-datasets       auth-datasets
-                                :auth-visualisations auth-visualisations
-                                :auth-dashboards     auth-dashboards
-                                :auth-collections    auth-collections})
+                             (load-auth-data dss rasters tenant-conn flow-api request collector tenant)
                              {:rasters             rasters
                               :auth-datasets       (mapv :id dss)
                               :auth-visualisations (mapv :id (all-visualisations tenant-conn))


Improve tree

Improve tree

--- src/play_clj/ui.clj
+++ src/play_clj/ui.clj
@@ -6,8 +6,8 @@
            [com.badlogic.gdx.scenes.scene2d Actor Stage]
            [com.badlogic.gdx.scenes.scene2d.ui ButtonGroup CheckBox Dialog
             HorizontalGroup Image ImageButton ImageTextButton Label ScrollPane
-            SelectBox Skin Slider Table TextButton TextField VerticalGroup
-            WidgetGroup Window]
+            SelectBox Skin Slider Table TextButton TextField Tree Tree$Node
+            VerticalGroup WidgetGroup Window]
            [com.badlogic.gdx.scenes.scene2d.utils ActorGestureListener Align
             ChangeListener ClickListener DragListener FocusListener
             NinePatchDrawable SpriteDrawable TextureRegionDrawable
@@ -97,12 +97,43 @@
   (cond
     (map? child)
     (.add ^Table (:object parent) ^Actor (:object child))
-    (vector? child)
+    (coll? child)
     (apply cell! (add-to-group! [parent (first child)]) (rest child))
     (keyword? child)
     (case child
       :row (.row ^Table (:object parent))
       nil)))
+
+(defn ^:private create-tree-node
+  [child]
+  {:object (Tree$Node. ^Actor (:object child))})
+
+(defn ^:private add-tree-nodes
+  [parent children]
+  (when-let [node (add-to-group! [parent (first children)])]
+    (doseq [child (rest children)]
+      (add-to-group! [node child]))
+    node))
+
+(defmethod add-to-group! Tree
+  [[parent child]]
+  (cond
+    (map? child)
+    (let [node (create-tree-node child)]
+      (.add ^Tree (:object parent) ^Tree$Node (:object node))
+      node)
+    (coll? child)
+    (add-tree-nodes parent child)))
+
+(defmethod add-to-group! Tree$Node
+  [[parent child]]
+  (cond
+    (map? child)
+    (let [node (create-tree-node child)]
+      (.add ^Tree$Node (:object parent) ^Tree$Node (:object node))
+      node)
+    (coll? child)
+    (add-tree-nodes parent child)))

 (defn add!
   [group & children]
@@ -311,6 +342,20 @@
   [entity k & options]
   `(u/call! ^TextField (:object ~entity) ~k ~@options))

+(defn tree*
+  [children arg]
+  (create-group (Tree. arg) children))
+
+(defmacro tree
+  [children arg & options]
+  `(let [entity# (tree* ~children ~arg)]
+     (u/calls! ^Tree (:object entity#) ~@options)
+     entity#))
+
+(defmacro tree!
+  [entity k & options]
+  `(u/call! ^Tree (:object ~entity) ~k ~@options))
+
 (defn vertical*
   [children]
   (create-group (VerticalGroup.) children))
@@ -326,12 +371,12 @@
   `(u/call! ^VerticalGroup (:object ~entity) ~k ~@options))

 (defn window*
-  [^String title arg children]
+  [children ^String title arg]
   (create-group (Window. title arg) children))

 (defmacro window
-  [title arg children & options]
-  `(let [entity# (window* ~title ~arg ~children)]
+  [children title arg & options]
+  `(let [entity# (window* ~children ~title ~arg)]
      (u/calls! ^Window (:object entity#) ~@options)
      entity#))


Fix drawable

Fix drawable

--- src/play_clj/ui.clj
+++ src/play_clj/ui.clj
@@ -14,14 +14,16 @@
            [com.esotericsoftware.tablelayout Cell]))

 (defmacro drawable
-  "Internal use only"
+  "Returns a subclass of [BaseDrawable](http://libgdx.badlogicgames.com/nightlies/docs/api/com/badlogic/gdx/scenes/scene2d/utils/BaseDrawable.html)
+
+    (drawable :texture-region)"
   [k & options]
-  `(~(symbol (str u/main-package ".scenes.scene2d.ui."
+  `(~(symbol (str u/main-package ".scenes.scene2d.utils."
                   (u/key->pascal k) "Drawable."))
      ~@options))

 (defmacro style
-  "Returns a style object based on the keyword `k`
+  "Returns a style object whose class is determined by the keyword `k`

     (style :check-box)"
   [k & options]


enable autoload by default for :browser target

figwheel does reloads by default as well so the expectation is that
reloads happens always. the HUD also makes it appear like reloads are
happening even if it just displays build start/success.

--- src/main/shadow/build/targets/browser.clj
+++ src/main/shadow/build/targets/browser.clj
@@ -258,7 +258,10 @@
 (defn configure
   [state mode config]
   (let [{:keys [output-dir asset-path public-dir public-path] :as config}
-        (merge default-browser-config config)]
+        (-> (merge default-browser-config config)
+            (cond->
+              (not (false? (get-in config [:devtools :autoload])))
+              (assoc-in [:devtools :autoload] true)))]

     (-> state
         (assoc ::build/config config) ;; so the merged defaults don't get lost


paging over measurements with metadata lookup working.

--- src/kixi/hecuba/api/measurements.clj
+++ src/kixi/hecuba/api/measurements.clj
@@ -19,28 +19,53 @@
    [qbits.hayt :as hayt]
    ))

+(defn sensor-metadata-for [store sensor-id]
+  (let [{:keys [type device-id]} sensor-id]
+    (dbnew/with-session [session (:hecuba-session store)]
+      (first (dbnew/execute session
+                            (hayt/select
+                             :sensor_metadata
+                             (hayt/where [[= :device-id device-id]
+                                          [= :type type]])))))))
+
+(defn- mk-bounds-from [store type device-id]
+  (let [sm  (sensor-metadata-for store {:type type :device-id device-id})
+        {:keys [lower_ts upper_ts]} sm]
+    (vector (atom (or (tc/to-date-time lower_ts) (org.joda.time.DateTime. Long/MAX_VALUE) ))
+            (atom (or (tc/to-date-time upper_ts) (org.joda.time.DateTime. 0))))))
+
+(defn resolve-start-end [store type device-id start end]
+  (mapv tc/to-date-time
+       (if (not (and start end))
+         (let [sm (sensor-metadata-for store {:type type :device-id device-id})
+               _ (log/info "sm: " sm)
+               [lower upper] ((juxt :lower_ts :upper_ts) sm)]
+           [(or start lower)
+            (or end upper)])
+         [start end])))
+
 (defn all-measurements
   "Returns a sequence of all the measurements for a sensor
    matching (type,device-id). The sequence pages to the database in the
    background. The page size is a clj-time Period representing a range
    in the timestamp column. page size defaults to (clj-time/hours 1)"
-  [store sensor-id start end & [opts]]
-  (let [{:keys [type device-id]} sensor-id
-        {:keys [page] :or {page (t/hours 1)}} opts
-        start      (tc/to-date-time start)
-        end        (tc/to-date-time end)
-        next-start (t/plus start page)]
-    (dbnew/with-session [session (:hecuba-session store)]
-      (lazy-cat (dbnew/execute session
-                 (hayt/select :measurements
-                              (hayt/where [[= :device-id device-id]
-                                           [= :type type]
-                                           [= :month (m/get-month-partition-key start)]
-                                           [>= :timestamp start]
-                                           [< :timestamp next-start]]))
-                 nil)
-       (when (t/before? next-start end)
-         (all-measurements store sensor-id next-start end opts))))))
+  ([store sensor-id & [opts]]
+     (let [{:keys [type device-id]} sensor-id
+           {:keys [page start end] :or {page (t/hours 1)}} opts
+           [start end] (resolve-start-end store type device-id start end )
+           _ (log/info "s: " start ", e:" end)
+           next-start (t/plus start page)]
+       (dbnew/with-session [session (:hecuba-session store)]
+         (lazy-cat (dbnew/execute session
+                                  (hayt/select :measurements
+                                               (hayt/where [[= :device-id device-id]
+                                                            [= :type type]
+                                                            [= :month (m/get-month-partition-key start)]
+                                                            [>= :timestamp start]
+                                                            [< :timestamp next-start]]))
+                                  nil)
+                   (when (t/before? next-start end)
+                     (all-measurements store sensor-id (merge opts {:start next-start :end end}))))))))

 (defn measurements-slice-handle-ok [store store-new ctx]
   (let [request                (:request ctx)
@@ -61,30 +86,19 @@
                                                           (dissoc :month :metadata :device-id)
                                                           util/camelify))))})))

-(defn sensor-metadata-for [store sensor-id]
-  (let [{:keys [type device-id]} sensor-id]
-    (dbnew/with-session [session (:hecuba-session store)]
-      (first (dbnew/execute session
-                            (hayt/select
-                             :sensor_metadata
-                             (hayt/where [[= :device-id device-id]
-                                          [= :type type]])))))))

-(defn- mk-bounds-from [store type device-id]
-  (let [sm  (sensor-metadata-for store {:type type :device-id device-id})
-        {:keys [lower_ts upper_ts]} sm]
-    (vector (atom (or lower_ts (java.util.Date. Long/MAX_VALUE) ))
-            (atom (or upper_ts (java.util.Date. 0))))))

-(defn- min-date [d1 d2]
-  (if (pos? (compare d1 d2))
-    d2
-    d1))

-(defn- max-date [d1 d2]
-  (if (pos? (compare d1 d2))
-    d1
-    d2))
+
+(defn- min-date [dt1 dt2]
+  (let [dt1' (tc/to-date-time dt1)
+        dt2' (tc/to-date-time dt2)]
+    (if (t/before? dt1' dt2') dt1' dt2')))
+
+(defn- max-date [dt1 dt2]
+  (let [dt1' (tc/to-date-time dt1)
+        dt2' (tc/to-date-time dt2)]
+    (if (t/before? dt1' dt2') dt2' dt1')))

 (defn index-post! [store store-new queue ctx]
   (let [{:keys [commander querier]} store


Made Eastwood happy.

Made Eastwood happy.

--- src/lambda_blog/templates/recent.clj
+++ src/lambda_blog/templates/recent.clj
@@ -36,7 +36,7 @@
            (tags/footer
             (hr))))

-(defn filtered-articles [filter {:keys [entries] :as ent}]
+(defn filtered-articles [entry-filter {:keys [entries] :as ent}]
   [(doctype :html)
    (html (header ent)
          (body (navigation ent)
@@ -45,7 +45,7 @@
                              (div {:class :container}
                                   (banner ent)
                                   (map entry-summary
-                                       (filter entries))
+                                       (entry-filter entries))
                                   (div {:class :well}
                                        (div {:class :row}
                                             (div {:class :text-center}


making host/port of the standalone server configurable

--- standalone/com/ig/swagger/search/standalone.clj
+++ standalone/com/ig/swagger/search/standalone.clj
@@ -12,8 +12,10 @@
 (defn start [system-atom config]
   (let [system (search/system-start config)]
     (reset! system-atom (assoc system
-                          :jetty (jetty/run-jetty (:routes system) {:port  3000
-                                                                    :join? false})))))
+                          :jetty (jetty/run-jetty (:routes system)
+                                                  {:port  (get-in config [:standalone :port] 3000)
+                                                   :host  (get-in config [:standalone :host] "0.0.0.0")
+                                                   :join? false})))))

 (defn stop [system-atom]
   (when-let [system @system-atom]
@@ -52,7 +54,4 @@

         config (find-config-file)]
     (add-libs-to-classpath home-dir)
-    (start the-system config)
-    (let [value (Object.)]
-      (locking value
-        (.await value)))))+    (start the-system config)))


First pass of relocating storage functionality

This moves the functionality from postgres-impl in to the tentacle
itself -- a tentacle will now know how to load/save/route itself. Rough,
but wanted to capture all the parts.

--- src/doctopus/doctopus/tentacle.clj
+++ src/doctopus/doctopus/tentacle.clj
@@ -2,18 +2,85 @@
   "A `Tentacle' defines a single unit of documentation -- one 'source' worth,
   and all the configs required to build it."
   (:require [clojure.string :as str]
-            [compojure.core :refer [GET routes context]]
-            [doctopus.configuration :refer [docs-uri-prefix]]
-            [doctopus.shell :refer [make-html-from-vec git-clone]]
-            [doctopus.storage :refer [save-to-storage load-from-storage count-records-for-tentacle backend]]
+            [compojure.core :refer [GET routes]]
+            [doctopus
+             [configuration :as configuration :refer [docs-uri-prefix]]
+             [db :as db]
+             [files :as f]
+             [shell :refer [git-clone make-html-from-vec]]]
             [me.raynes.fs :as fs]
-            [taoensso.timbre :as log]))
+            [ring.util
+             [mime-type :as ring-mime]
+             [response :as response]]
+            [taoensso.timbre :as log])
+  (:import java.io.ByteArrayInputStream))

+;; To match against mimetype strings
+(def is-img-regex #"^image")
+
+;; Utilities for marking HTTP Resposnes
+(defn- assign-body-and-image
+  [base-doc doc-file mime-type]
+  (log/debug base-doc)
+  (let [doc-content (slurp doc-file)]
+    (if (re-find is-img-regex mime-type)
+      (assoc base-doc :body "image" :image (.getBytes doc-content))
+      (assoc base-doc :body doc-content :image nil))))
+
+(defn- build-body
+  "Examine a DB result to see if it's an image; if so, build a ByteArrayInputStream
+  and use it as the response body."
+  [res]
+  (if (= "image" (:body res))
+    (let [bais (ByteArrayInputStream. (:image res))]
+      (assoc res :body bais))
+    res))
+
+(defn- make-response
+  [res]
+  (-> (response/response (:body res))
+      (response/content-type (:mime-type res))))
+
+(defn- load-fn
+  [uri]
+  (if-let [res (first (db/get-document-by-uri uri))]
+    (-> res
+        (build-body)
+        (make-response))))
+
+;; Acquiring repos
 (defn- get-source
   "We may eventually support more ways of getting source! Would be nice. Not
   today."
   [sc-location dest]
   (git-clone sc-location dest))
+
+;; Storing generated HTML
+(defn- save-to-database
+  "Load every document from a directory in to the database.
+
+  Note: at time of writing, at the end of load the call to update-document-index
+  will recalculate the _entire_ `search_vector' column in PG. If this ever gets
+  too costly, it can be improved, probably by adding additional info to the
+  update's `where' clause.
+
+  Also note: there are two kinds of files to be loaded: text and images (binary)."
+  [tent-name src-dir]
+  (let [relpath-file-pairs (f/list-files-with-relative-paths src-dir)]
+    (doseq [[relpath doc-file] relpath-file-pairs
+            :let [doc-mime-type (ring-mime/ext-mime-type relpath)
+                  base-doc {:name (str tent-name "-" (.getName doc-file))
+                            ;; :path doc-file
+                            :mime-type doc-mime-type
+                            :uri (str "/" configuration/docs-uri-prefix "/" tent-name relpath)
+                            :tentacle-name tent-name}]]
+      (if (nil? doc-mime-type)
+        (log/warn "Refusing to save" relpath ", cannot discern mime-type")
+        (let [new-doc (assign-body-and-image base-doc doc-file doc-mime-type)]
+          (log/debug "Reading doc from:" relpath)
+          (log/debug "Giving doc URI:" (:uri new-doc))
+          (db/save-document! new-doc))))
+    (db/update-document-index)))

 ;; #### "Try, report, return"-ing
 ;; I've found myself doing a lot of this:
@@ -42,16 +109,17 @@
 ;; TENTACLES
 (defprotocol TentacleMethods
   (load-html [this] "Makes sure the HTML has been generated for this tentacle")
-  (generate-html [this])
-  (save-build-output [this dir])
-  (get-html-entrypoint [this])
-  (generate-routes [this]))
+  (generate-html [this]) ;; check
+  (save-build-output [this dir]) ;; check
+  (get-html-entrypoint [this]) ;; check
+  (generate-routes [this]) ;; check
+  (count-records [this])) ;; check

 (defrecord Tentacle
     [name html-commands output-root source-location entry-point]
   TentacleMethods
   (load-html [this]
-    (if (= 0 (count-records-for-tentacle backend this))
+    (if (= 0 (count-records this))
       (do (log/info "No HTML found for" (:name this))
           (generate-html this))
       (log/info "Found html for" (:name this))))
@@ -70,11 +138,14 @@
   (save-build-output [this dir]
     (let [{:keys [name]} this]
       (check-and-report
-       (save-to-storage backend name dir) name "saved HTML" "save HTML")))
+       (save-to-database name dir) name "saved HTML" "save HTML")))
   (get-html-entrypoint [this]
     (str/join "/" ["" docs-uri-prefix (:name this) (:entry-point this)]))
   (generate-routes [this]
     (routes
      (GET "*" {:keys [uri]}
-          (log/debug "Looking for URI:" uri)
-          (load-from-storage backend uri)))))
+          (if-let [res (first (db/get-document-by-uri uri))]
+            (-> res
+                (build-body)
+                (make-response))))))
+  (count-records [this] (count (db/get-all-documents-for-tentacle (:name this)))))


polish ab.clj

polish ab.clj

--- ann/src/ann/ab.clj
+++ ann/src/ann/ab.clj
@@ -24,17 +24,13 @@

 (defn good-enough?
   [delta]
-  (let [result (< (abs delta) 0.9)]
-    (log "Good-enough?" delta result)
-    result))
+  (< (abs delta) 0.9))

 (defn calc
   "w0 + w1*x1 + w2*x2"
   [inputs weights]
-  (let [result (+ (apply + (map * inputs (rest weights)))
-                  (first weights))]
-    (log "Calc:" inputs weights result)
-    result))
+  (+ (apply + (map * inputs (rest weights)))
+     (first weights)))

 ;;; 感知器

@@ -45,14 +41,12 @@
 (defn train-perceptron
   "训练感知器，返回新的感知器"
   [perceptron inputs target]
-  (let [delta (- target (calc inputs perceptron))
-        new-perceptron (map (fn [input weight]
-                              (+ weight
-                                 (* delta (sgn input) study-rate)))
-                            (concat [1] inputs)
-                            perceptron)]
-    (log "New Perceptron:" new-perceptron)
-    new-perceptron))
+  (let [delta (- target (calc inputs perceptron))]
+    (map (fn [input weight]
+           (+ weight
+              (* delta (sgn input) study-rate)))
+         (concat [1] inputs)
+         perceptron)))

 ;;; Test

@@ -69,24 +63,23 @@
                    (good-enough? (- target (calc inputs perceptron))))
                  data-collection)))

-  (def my-perceptron (loop [perceptron (create-perceptron)]
-                       (let [my-perc (loop [perc perceptron, data-collection data-collection]
+  (def perceptron (loop [perceptron (create-perceptron)]
+                       (let [perceptron (loop [perceptron perceptron, data-collection data-collection]
                                        (if (first data-collection)
                                          (let [{inputs :inputs target :target} (first data-collection)]
-                                           (recur (train-perceptron perc inputs target) (rest data-collection)))
-                                         perc))]
-                         (if (all-good-enough? my-perc)
-                           my-perc
-                           (recur my-perc)))))
+                                           (recur (train-perceptron perceptron inputs target) (rest data-collection)))
+                                         perceptron))]
+                         (if (all-good-enough? perceptron)
+                           perceptron
+                           (recur perceptron)))))

-  (log my-perceptron)
+  (log perceptron)

   ;; Test

   (defn gen
     [inputs]
-    (println (sgn (calc inputs my-perceptron)))
-    )
+    (log "TEST" inputs "=>" (sgn (calc inputs perceptron))))

   (gen [1 1])
   (gen [1 -1])


adding some new resolution functions

adding some new resolution functions

--- support/src/figwheel/client/file_reloading.cljs
+++ support/src/figwheel/client/file_reloading.cljs
@@ -1,6 +1,7 @@
 (ns figwheel.client.file-reloading
   (:require
    [goog.Uri :as guri]
+   [goog.string]
    [goog.net.jsloader :as loader]
    [clojure.string :as string]
    [cljs.core.async :refer [put! chan <! map< close! timeout alts!] :as async])
@@ -23,9 +24,13 @@
 (defn get-meta-data-for-ns [ns]
   (get ns-meta-data ns))

+(defn resolve-ns [ns]
+  (str (string/replace-first (.-basePath js/goog) "/goog" "")
+       (string/replace ns "." "/")
+       ".js"))
+
 (defn js-reload [{:keys [request-url namespace dependency-file meta-data] :as msg} callback]
   (swap! ns-meta-data assoc namespace meta-data)
-  (print (pr-str @ns-meta-data))
   (if (and
        (or dependency-file
            (and meta-data (:figwheel-load meta-data))
@@ -79,8 +84,7 @@
 (defn truncate-url [url]
   (-> (first (string/split url #"\?"))
       (string/replace-first (str (.-protocol js/location) "//") "")
-      (string/replace-first "http://" "")
-      (string/replace-first "https://" "")
+      (string/replace-first ".*://" "")
       (string/replace-first #"^//" "")
       (string/replace-first #"[^\/]*" "")))

@@ -88,6 +92,9 @@
   (let [trunc-href (truncate-url link-href)]
     (or (= file trunc-href)
         (= (truncate-url request-url) trunc-href))))
+
+#_(defn matches-file-new? [{:keys [file] } link-href]
+    (.endsWith goog.string file (truncate-url link-href)))

 (defn get-correct-link [f-data]
   (some (fn [l]


Split benchmarking code into two, later to be utilized by SQL benchmarks

--- taxi-rides-clj/src/taxi_rides_clj/benchmark.clj
+++ taxi-rides-clj/src/taxi_rides_clj/benchmark.clj
@@ -146,22 +146,14 @@

 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

-(defn benchmark-es [filter-generators aggregations n-tries n-parallels]
-  (let [percentiles (partial percentiles [5 10 25 50 75 90 95])
-        client    (make-client)
-        benchmark (fn [filter-generator aggregation]
-                    (let [t-start (System/nanoTime)
-                          result  (->> (filter-generator) aggregation (run-query client))
-                          t-end   (System/nanoTime)]
-                      {:took (- t-end t-start)
-                     ; :result result
-                       :hits (-> result :body :hits :total)}))]
+(defn run-benchmark [benchmark-fn filter-generators aggregations n-tries n-parallels]
+  (let [percentiles (partial percentiles [5 10 25 50 75 90 95])]
     (doall (for [n-parallel                     n-parallels
                  [filter-name filter-generator] filter-generators
                  [aggregation-name aggregation] aggregations]
              (let [_         nil ; (u/my-println "Started " [filter-name aggregation-name n-parallel])
                    t-start   (System/nanoTime)
-                   results   (doall (cp/upfor n-parallel [i (range n-tries)] (benchmark filter-generator aggregation)))
+                   results   (doall (cp/upfor n-parallel [i (range n-tries)] (benchmark-fn filter-generator aggregation)))
                    took      (-> (System/nanoTime) (- t-start) (* 0.000001) Math/round)
                    avg       (/ took n-tries 1.0)
                    _         (u/my-println "Finished " [filter-name aggregation-name n-parallel] " in " took " ms, avg " avg " ms")
@@ -173,6 +165,18 @@
                 :aggs          aggregation-name
                 :avg           avg
                 :stats         (->> [p-took p-hits] (map vals) (apply zipmap) (into (sorted-map)))})))))
+
+
+(defn benchmark-es [& args]
+  (let [client       (make-client)
+        benchmark-fn (fn [filter-generator aggregation]
+                       (let [t-start (System/nanoTime)
+                             result  (->> (filter-generator) aggregation (run-query client))
+                             t-end   (System/nanoTime)]
+                         {:took (- t-end t-start)
+                        ; :result result
+                          :hits (-> result :body :hits :total)}))]
+    (apply run-benchmark benchmark-fn args)))

 (comment
   (do (println "")


remove unneeded test file

remove unneeded test file

--- test/clj_money/schema_test.clj
+++ test/clj_money/schema_test.clj
@@ -1,23 +0,0 @@
-(ns clj-money.schema-test
-  (:require [clojure.test :refer :all]
-            [clojure.pprint :refer [pprint]]
-            [clojure.data :refer [diff]]
-            [schema.core :as s])
-  (:use [clj-money.schema]))
-
-(def Thing
-  "Test model for validating"
-  {:name s/Str
-   :id (s/pred (partial re-matches #"[A-Z]\d{2}") "invalid format")
-   :size s/Int})
-
-(deftest user-friendly-validation-messages
-  (try
-    (s/validate Thing {:id "invalid" :size "large"})
-    (catch clojure.lang.ExceptionInfo e
-      (let [data (ex-data e)
-            expected {:name "is required"
-                      :id "is not valid"
-                      :size "must be a number"}
-            actual (user-friendify data)]
-        (is (= expected actual))))))


using pmap to speed things up a bit

using pmap to speed things up a bit

--- src/statuspage_crawler/domain/crawler.clj
+++ src/statuspage_crawler/domain/crawler.clj
@@ -54,9 +54,9 @@
          (into {}))))

 (defn images-in-urls [job-id urls]
-  (let [url->images (reduce (fn [url->imgs url]
-                              (merge url->imgs
-                                     (find-all-image-links-in-url url)))
-                            {} urls)]
+  (let [url->images (->> urls
+                         (pmap (fn [url]
+                                 (find-all-image-links-in-url url)))
+                         (apply merge))]
     (db/update-job job-id {:body (json/encode url->images)
                            :status "completed"})))


added tests back for tabular sugar

added tests back for tabular sugar

--- test/midje/ideas/t_tabular.clj
+++ test/midje/ideas/t_tabular.clj
@@ -25,6 +25,37 @@
    (+ ?a ?b) => ?result)
  ?a    ?b      ?result
  1     2       3)
+
+(tabular
+ (fact "will ignore optional pipes separating table columns"
+   (str ?a ?b ?c) => ?result)
+
+ ?a  | ?b  | ?c  | ?result
+ "a" | "|" | "c" | "a|c" )
+
+(tabular
+ (fact "will ignore an optional ':where' above the table"
+   (str ?a ?b) => ?result)
+
+ :where
+ ?a     | ?b     | ?result
+ 'where | :where | "where:where") ;; just to makes sure
+
+(tabular
+ (fact "will ignore an optional 'where' above the table"
+   (+ ?a ?b) => ?result)
+
+ where
+ ?a | ?b | ?result
+ 1  | 2  | 3)
+
+(tabular
+ (fact "can have different forms of where or pipe in the data, no problem"
+   (str ?a ?b ?c ?d) => ?result)
+
+ where
+ ?a     | ?b  | ?c     | ?d | ?result
+ :where | "|" | 'where | '| | ":where|where|")

  ;; Error handling


convert pipe/where tabular tests to hav no ?'s to make sure those cases were covered in the general (non-?) case

--- test/midje/ideas/t_tabular.clj
+++ test/midje/ideas/t_tabular.clj
@@ -35,8 +35,8 @@
     (fact (+ a b) => result )

        a  b      result
-       c  2      3       ;; need second row, else this mis-parsing
-       4  5      9))     ;; tabular will think there are 0 rows
+       c  2      3       ;; need second row, else (in the failure case) this
+       4  5      9))     ;; mis-parsing tabular will think there are 0 rows

 (defn f? [x] true)

@@ -61,33 +61,33 @@

 (tabular
  (fact "will ignore optional pipes separating table columns"
-   (str ?a ?b ?c) => ?result)
-
- ?a  | ?b  | ?c  | ?result
+   (str a b c) => result)
+
+ a   | b   | c   | result
  "a" | "|" | "c" | "a|c" )

 (tabular
  (fact "will ignore an optional ':where' above the table"
-   (str ?a ?b) => ?result)
+   (str a b) => result)

  :where
- ?a     | ?b     | ?result
+ a      | b      | result
  'where | :where | "where:where") ;; just to makes sure

 (tabular
  (fact "will ignore an optional 'where' above the table"
-   (+ ?a ?b) => ?result)
+   (+ a b) => result)

  where
- ?a | ?b | ?result
+ a  |  b | result
  1  | 2  | 3)

 (tabular
  (fact "can have different forms of where or pipe in the data, no problem"
-   (str ?a ?b ?c ?d) => ?result)
+   (str a b c d) => result)

  where
- ?a     | ?b  | ?c     | ?d | ?result
+ a      | b   | c      | d  | result
  :where | "|" | 'where | '| | ":where|where|")


[#253] RuntimeException wasn't suitable for non-REPL usage (#258)

--- src/selmer/tags.clj
+++ src/selmer/tags.clj
@@ -399,7 +399,7 @@
     (let [edn->html @(resolve 'json-html.core/edn->html)]
       (fn [ctx-map]
         (edn->html ctx-map)))
-    (catch java.lang.RuntimeException _
+    (catch java.lang.Exception _
       basic-edn->html)))

 (defn debug-handler [_ _ _ _]


removed unused function

removed unused function

--- src/selmer/util.clj
+++ src/selmer/util.clj
@@ -134,11 +134,6 @@
       (.getContextClassLoader)
       (.getResource resource)))

-(def decode-path
-  (memoize
-    (fn [file-path]
-      (java.net.URLDecoder/decode file-path "utf-8"))))
-
 (defn resource-path [template]
   (if-let [path @custom-resource-path]
     (java.net.URL. (str path template))


Migrate to google custom search instead of dead image search.

--- src/yetibot/core/models/google_search.clj
+++ src/yetibot/core/models/google_search.clj
@@ -1,20 +1,27 @@
 (ns yetibot.core.models.google-search
   (:require
+    [yetibot.core.config :refer [get-config conf-valid?]]
     [yetibot.core.util.http :refer [get-json map-to-query-string]]))

-(def endpoint "http://ajax.googleapis.com/ajax/services/search/images")
-(def configured? true)
+(def config (get-config :yetibot :models :google-search))
+(def endpoint "https://www.googleapis.com/customsearch/v1")
+(def configured? (conf-valid? config))

-(def ^:private format-result (juxt :url :contentNoFormatting))
+(defn- format-result [result]
+  [(-> (get-in result [:pagemap :cse_image])
+       rand-nth
+       :src)
+   (:title result)
+   (:link result)])

 (defn- fetch-image [q n]
   (let [uri (str endpoint "?" (map-to-query-string
-                                {:v "1.0" :rsz n
-                                 :q q :safe "active"}))]
+                                {:key (:key config) :cx (:cx config)
+                                 :num n :q q :safe "high"}))]
     (get-json uri)))

 (defn image-search
   ([q] (image-search q 8))
   ([q n]
    (map format-result
-        (-> (fetch-image q n) :responseData :results))))
+        (-> (fetch-image q n) :items))))


user refactor

user refactor

--- dev/user.clj
+++ dev/user.clj
@@ -10,7 +10,7 @@
 (set! *unchecked-math* :warn-on-boxed)

 (def http-handler
-  (wrap-reload #'swarmpit.server/handler))
+  (wrap-reload #'swarmpit.server/app))

 (defn run []
   (figwheel/start-figwheel!))


Add hints to RepositoryConnection implementation of ITripleReadable.

Issue #161 - Add type hints to local bindings within the
implementation of the ITripleReadable protocol for
RepositoryConnection. This ensures the expected overload of
getStatements is chosen.

--- src/grafter_2/rdf4j/repository.clj
+++ src/grafter_2/rdf4j/repository.clj
@@ -6,13 +6,13 @@
             [grafter-2.rdf4j.formats :as format]
             [grafter-2.rdf4j.io :as rio]
             [me.raynes.fs :as fs])
-  (:import [org.eclipse.rdf4j.model Resource Statement URI]
+  (:import [org.eclipse.rdf4j.model Resource Statement URI IRI Value]
            [org.eclipse.rdf4j.query BindingSet BooleanQuery GraphQuery Query QueryLanguage TupleQuery Update]
            [org.eclipse.rdf4j.repository Repository RepositoryConnection]
-           [org.eclipse.rdf4j.sail.inferencer.fc CustomGraphQueryInferencer DirectTypeHierarchyInferencer ForwardChainingRDFSInferencer])
+           [org.eclipse.rdf4j.sail.inferencer.fc CustomGraphQueryInferencer DirectTypeHierarchyInferencer ForwardChainingRDFSInferencer]
+           [org.eclipse.rdf4j.common.iteration CloseableIteration Iteration])
   (:import grafter_2.rdf.SPARQLRepository
-           grafter_2.rdf.protocols.IStatement
-           org.eclipse.rdf4j.common.iteration.CloseableIteration
+           grafter_2.rdf.protocols.IStatement
            org.eclipse.rdf4j.model.impl.URIImpl
            org.eclipse.rdf4j.query.impl.DatasetImpl
            org.eclipse.rdf4j.repository.event.base.NotifyingRepositoryWrapper
@@ -586,12 +586,15 @@
 (extend-type RepositoryConnection
   pr/ITripleReadable
   (pr/to-statements [this {:keys [:grafter-2.repository/infer] :or {infer true}}]
-    (let [f (fn next-item [i]
+    (let [f (fn next-item [^Iteration i]
               (when (.hasNext i)
                 (let [v (.next i)]
-                  (lazy-seq (cons (rio/backend-quad->grafter-quad v) (next-item i))))))]
-      (let [iter (.getStatements this nil nil nil infer (into-array Resource []))]
-        (f iter)))))
+                  (lazy-seq (cons (rio/backend-quad->grafter-quad v) (next-item i))))))
+          ^Resource subj nil
+          ^IRI pred nil
+          ^Value obj nil
+          iter (.getStatements this subj pred obj (boolean infer) (resource-array))]
+      (f iter))))

 (defn shutdown
   "Cleanly shutsdown the repository."


making the error code context scrollable

--- support/src/figwheel/client/heads_up.cljs
+++ support/src/figwheel/client/heads_up.cljs
@@ -172,8 +172,8 @@

 (defn format-inline-error [inline-error]
   (let [lines (map format-inline-error-line (pad-line-numbers inline-error))]
-    (str "
"
+    (str "
"
          (string/join "\n" lines)
          "
")))


Turn off signing for now

Turn off signing for now

--- profiles.clj
+++ profiles.clj
@@ -1,10 +1,11 @@
 {
  :user {
+   :repositories [["clojars" {:sign-releases false}]]
    :plugins [
              ;; [lein-clojars "0.9.0"]
              ;; [lein-light "0.0.4"]
              ;; [lein-difftest "2.0.0"]
-             ;; [lein-grep "0.1.0"]
+             ;; [lein-grep "0.1.1"]
              ;; [lein-open "0.1.0"]
              ;; [lein-outdated "1.0.0"]
              ;; [lein-exec "0.3.0"]


lint fix.

lint fix.

--- src/metabase/task/sync_databases.clj
+++ src/metabase/task/sync_databases.clj
@@ -43,4 +43,3 @@
                                      (cron/schedule (cron/cron-schedule "0 0 0 * * ? *")))))
   ;; submit ourselves to the scheduler
   (task/schedule-task! @sync-databases-job @sync-databases-trigger))
-


(PE-2741) fix broken test

(PE-2741) fix broken test

--- test/puppetlabs/trapperkeeper/services_test.clj
+++ test/puppetlabs/trapperkeeper/services_test.clj
@@ -276,8 +276,9 @@
     (is (thrown-with-msg?
           Exception
           #"Incorrect macro usage"
-          (macroexpand '(service Service1
-                                 []
-                                 (service1-fn
-                                   "This is an example of an invalid docstring"
-                                   [this] nil)))))))+          (macroexpand '(puppetlabs.trapperkeeper.services/service
+                          puppetlabs.trapperkeeper.services-test/Service1
+                          []
+                          (service1-fn
+                            "This is an example of an invalid docstring"
+                            [this] nil)))))))


typo in node(s)

typo in node(s)

--- test/clojure/digitalocean_expect/test.clj
+++ test/clojure/digitalocean_expect/test.clj
@@ -71,4 +71,4 @@
 (expect (< (count (filter large? (map :size_id nodes))) 2))

 ; check names match prescribed pattern
-(expect (every? valid-name? (map :name node)))
+(expect (every? valid-name? (map :name nodes)))


CMR-4375: updated cmr_metadata_preview gem to commit id: ab477bd.

--- collection-renderer-lib/project.clj
+++ collection-renderer-lib/project.clj
@@ -14,7 +14,7 @@
    the hardcoded commit id during dev integration with cmr_metadata_preview project.
    The hardcoded commit id should be updated when MMT releases a new version of the gem."
   (or (System/getenv "CMR_METADATA_PREVIEW_COMMIT")
-      "6b51c3fc0ae"))
+      "ab477bd"))

 (def gem-install-path
   "The directory within this library where Ruby gems are installed."


remove unnecessary type

remove unnecessary type

--- src/tween/core.cljs
+++ src/tween/core.cljs
@@ -107,11 +107,11 @@
                           :x 100
                           :y 100}
                    :fps-ticker 60
-                   :movement [{:type :show :item 1 :axis :x :pos 400 :t 1000 :ease 4}
-                              {:type :show :item 1 :axis :y :pos 75 :t 500 :ease 2 :alpha 0}
-                              {:type :move :item 1 :axis :y :pos 125 :t 100 :alpha 0}
-                              {:type :move :item 1 :axis :y :pos 100 :t 500 :ease 2 :alpha 1}
-                              {:type :move :item 1 :axis :x :pos 100 :t 800 :ease 2}]}]])
+                   :movement [{:item 1 :axis :x :pos 400 :t 1000 :ease 4}
+                              {:item 1 :axis :y :pos 75 :t 500 :ease 2 :alpha 0}
+                              {:item 1 :axis :y :pos 125 :t 100 :alpha 0}
+                              {:item 1 :axis :y :pos 100 :t 500 :ease 2 :alpha 1}
+                              {:item 1 :axis :x :pos 100 :t 800 :ease 2}]}]])

 ;; -------------------------
 ;; Initialize app


add support for utils classes query

add support for utils classes query

--- src/app/cli.clj
+++ src/app/cli.clj
@@ -15,7 +15,7 @@
                ["-i" "--interactive" "launch interactive mode" :flag true :default false]
                ["-l" "--linter" "launch linter mode" :flag true :default false]
                ["-d" "--dir DIRNAME" "REQUIRED: Directory containing the code to check"]
-               ["-q" "--query QUERYNAME" "REQUIRED: Query to perform: mc=many constructors, mcp=many constructor parameters, st=singleton type"]
+               ["-q" "--query QUERYNAME" "REQUIRED: Query to perform: mc=many constructors, mcp=many constructor parameters, st=singleton type, u=utils classes"]
                ["-t" "--threshold VALUE" "Threshold to be used in the query" :default 0
                 :parse-fn #(Integer/parseInt %)
                 :validate [#(>= % 0) "Must be a number equal or greater to 0"]]
@@ -26,6 +26,7 @@
     (= "mc" name) classesWithManyConstructorsOp
     (= "mcp" name) constructorsWithManyParametersOp
     (= "st" name) classesAndSingletonTypeOp
+    (= "u" name) utilsClassesOp
     :else nil))

 (defn run [opts]


Remove commented code.

Remove commented code.

--- test/uxbox/tests/test_auth.clj
+++ test/uxbox/tests/test_auth.clj
@@ -11,58 +11,6 @@
             [uxbox.tests.helpers :as th]))

 (t/use-fixtures :each th/database-reset)
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Services Tests
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-
-;; (t/deftest test-success-auth
-;;   (let [user-data {:username "user1"
-;;                    :fullname "user 1"
-;;                    :password  (hashers/encrypt "user1")
-;;                    :email "user1@uxbox.io"}
-;;         user (with-open [conn (up/get-conn)]
-;;                (usu/create-user conn user-data))
-;;         data {:type :auth/login
-;;               :scope "foobar"
-;;               :username "user1"
-;;               :password "user1"}
-;;         result @(usv/novelty data)]
-;;     (t/is (contains? result :token))
-;;     (t/is (string? (:token result)))))
-
-;; (t/deftest test-success-by-email
-;;   (let [user-data {:username "user1"
-;;                    :password  (hashers/encrypt "user1")
-;;                    :fullname "user 1"
-;;                    :email "user1@uxbox.io"}
-;;         user (with-open [conn (up/get-conn)]
-;;                (usu/create-user conn user-data))
-;;         data {:type :auth/login
-;;               :username "user1@uxbox.io"
-;;               :fullname "user 1"
-;;               :password "user1"}
-;;         result @(usv/novelty data)]
-;;     (t/is (contains? result :token))
-;;     (t/is (string? (:token result)))))
-
-;; (t/deftest test-failed-auth
-;;   (let [user-data {:username "user2"
-;;                    :fullname "user 2"
-;;                    :password  (hashers/encrypt "user2")
-;;                    :email "user2@uxbox.io"}
-;;         user (with-open [conn (up/get-conn)]
-;;                (usu/create-user conn user-data))
-;;         data {:type :auth/login
-;;               :username "user1"
-;;               :password "user1"}
-;;         result (th/await (usv/novelty data))]
-;;     (t/is (th/ex-info? result))))
-
-
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
-;; Frontend Test
-;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

 (t/deftest test-http-success-auth
   (let [data {:username "user1"


Removing the reverse for lists, as Lee pointed out, order doesnt matter at that point.

--- src/clojush.clj
+++ src/clojush.clj
@@ -98,7 +98,7 @@
          shuf (vec coll)]
     (if (zero? i)
       (cond (vector? coll) shuf
-            (list? coll) (reverse (into (list) shuf))
+            (list? coll) (into (list) shuf)
             :else
             (seq shuf))
       (recur (dec i)


add wrong key replacement

add wrong key replacement

--- src/specly/core.clj
+++ src/specly/core.clj
@@ -90,7 +90,6 @@

 ;; suggest a completely different key if its a complex value and it conforms to
 ;; another key that isn't already in the map
-(defn complies-to-other-key [])

 (comment

@@ -107,6 +106,64 @@

   (spelling-suggestions   #{:asdf/asdf :asdf/abg :asdfffff/a}  :asdf/asd )
   )
+
+
+;; suggest a completely different key if its a complex value and it conforms to
+;; another key that isn't already in the map
+
+;; when a key val is unknown we can rank it against all the keys that are not take in the map
+
+(defn replacement-suggestions [{:keys [k->s known-keys]} map-x ky]
+  (when-let [unused-keys (not-empty (filter (complement map-x) known-keys))]
+    (let [key-val (get map-x ky)]
+      (not-empty
+       (keep
+        (fn [k]
+          (let [unused-spec (k->s k)]
+            (cond
+              (and (coll? key-val)
+                   (s/valid? unused-spec key-val))
+              [k 1]
+              (map? key-val)
+              (cond
+                (s/valid? unused-spec key-val)
+                [k 1]
+                (fuzzy-spec? unused-spec)
+                (let [score (fuzzy-conform-score (spec-from-registry unused-spec) key-val)]
+                  (when (> score 6/10)
+                    [k score]))))))
+        unused-keys)))))
+
+(defn replacement-suggestion [key-spec-data map-x ky]
+  (->> (replacement-suggestions key-spec-data map-x ky)
+       (sort-by (comp - second))
+       ffirst))
+
+(s/explain :fig-opt/build-config {:id "asdf", :source-paths [""], :assert 1})
+
+(comment
+
+  (replacement-suggestion (parse-keys-args
+                            :opt-un [:fig-opt/http-server-root
+                                     :fig-opt/server-port
+                                     :fig-opt/server-ip]
+                            :req-un [:fig-opt/build-config])
+                           {;:http-server-root 1
+                            :server-por 1
+                                        ;:asdf 1
+                                        ;:badly 3
+                            :asdf {:id "asdf"
+                                   :source-paths [""]
+                                   :assert true
+                                   ;:madly 2
+
+                                   }}
+                           :asdf
+                           )
+
+  )
+
+

 ;; thoughts about fuzzy conform
 ;; the reason for fuzzy conform is two fold
@@ -165,6 +222,7 @@
 (defn fuzzy-conform-score-helper [{:keys [known-keys  k->s] :as spec-key-data} map-x]
   (when (map? map-x)
     (let [key-to-score
+
           (fn [[k v]]
             (cond
               (map? v)
@@ -263,10 +321,18 @@
                                  (assoc
                                   ::misspelled-key unknown-key
                                   ::correct-key suggest))]
-                            [(conj pred-path :unknown-key unknown-key)
-                             (-> exp-data
-                                 #_(update-in [:in] conj unknown-key)
-                                 (assoc ::unknown-key unknown-key))]
+                            (if-let [replace-suggest (replacement-suggestion spec-key-data x unknown-key)]
+                              [(conj pred-path :misspelled-key unknown-key)
+                               (-> exp-data
+                                   #_(update-in [:in] conj unknown-key)
+                                   (assoc
+                                    ::misspelled-key unknown-key
+                                    ::correct-key suggest))]
+                              [(conj pred-path :unknown-key unknown-key)
+                               (-> exp-data
+                                   #_(update-in [:in] conj unknown-key)
+                                   (assoc ::unknown-key unknown-key))])
+
                             )))))))))
       ;; These can be improved
       (gen* [_ a b c]


Fix typo

Fix typo

--- src/search_clojure_archives/core.clj
+++ src/search_clojure_archives/core.clj
@@ -67,7 +67,7 @@
                                          "9200"))
         elasticsearch-url (format "http://%s:%d/" elasticsearch-host elasticsearch-port)]
     (try
-      (log/info 3#3 3#3 "Connecting to ElasticSearch server at" elasticsearch-url)
+      (log/info "Connecting to ElasticSearch server at" elasticsearch-url)
       (reset! es-url elasticsearch-url)
       (let [state (:status (es/cluster-stats @es-url))]
         (log/debug "ElasticSearch cluster state:" state)


test: var can be patched with a recorded fake

--- test/unit/patch.cljc
+++ test/unit/patch.cljc
@@ -183,6 +183,17 @@
     (is (= "bar" (funcs/sum 3 4)))))

 (u/-deftest
+  "var can be patched with a recorded fake"
+  (f/with-fakes
+    (f/patch! #'funcs/sum (f/recorded-fake [[(f/arg integer?) (f/arg integer?)] #(* %1 %2)]))
+
+    (is (= 2 (funcs/sum 1 2)))
+    (is (= 12 (funcs/sum 3 4)))
+
+    (is (f/was-called funcs/sum [1 2]))
+    (is (f/was-called funcs/sum [3 4]))))
+
+(u/-deftest
   "record instantiation using -> can be patched"
   (f/with-fakes
     (f/patch! #'->MyRecord (constantly 123))


Add :ns metadata to defassert/defcontext/deftest/defsuite

--- lazytest.clj
+++ lazytest.clj
@@ -81,7 +81,7 @@
   "Defines an Assertion.
   decl => docstring? [locals*] body*"
   [name & decl]
-  (let [m {:name name}
+  (let [m {:name name, :ns *ns*}
         m (if (string? (first decl)) (assoc m :doc (first decl)) m)
         decl (if (string? (first decl)) (next decl) decl)
         argv (first decl)
@@ -112,33 +112,11 @@
     (let [active (reduce close-context active (:parents c))]
       (dissoc active c))))

-;; (let [m {:name name}
-;;         m (if (string? (first decl)) (assoc m :doc (first decl)) m)
-;;         decl (if (string? (first decl)) (next decl) decl)
-;;         bindings (first decl)
-;;         assertions (next decl)]
-;;     (assert (vector? bindings))
-;;     (assert (even? (count bindings)))
-;;     (let [locals (vec (map first (partition 2 bindings)))
-;;           contexts (vec (map (comp coerce-context second) (partition 2 bindings)))]
-;;       (assert (every? symbol locals))
-;;       `(def ~name
-;;             (with-meta (TestCase ~contexts
-;;                                  ~(loop [r [], as assertions]
-;;                                     (if (seq as)
-;;                                       (if (string? (first as))
-;;                                         (recur (conj r `(with-meta (assertion ~locals ~(next as))
-;;                                                           {:doc ~(first as)}))
-;;                                                (nnext as))
-;;                                         (recur (conj r `(assertion ~locals ~(first as))) (next as)))
-;;                                       r)))
-;;               '~m))))
-
 (defmacro defcontext
   "Defines a context.
   decl => docstring? [bindings*] before-body* (:after [state] after-body*)?"
   [name & decl]
-  (let [m {:name name}
+  (let [m {:name name, :ns *ns*}
         m (if (string? (first decl)) (assoc m :doc (first decl)) m)
         decl (if (string? (first decl)) (next decl) decl)
         bindings (first decl)
@@ -206,7 +184,7 @@
   binding => symbol context
   assertion => docstring? expression"
   [name & decl]
-  (let [m {:name name}
+  (let [m {:name name, :ns *ns*}
         m (if (string? (first decl)) (assoc m :doc (first decl)) m)
         decl (if (string? (first decl)) (next decl) decl)
         bindings (first decl)
@@ -232,7 +210,7 @@
   "Defines a test suite containing other test cases or suites.
   decl => docstring? [context*] children*"
   [name & decl]
-  (let [m {:name name}
+  (let [m {:name name, :ns *ns*}
         m (if (string? (first decl)) (assoc m :doc (first decl)) m)
         decl (if (string? (first decl)) (next decl) decl)
         contexts (first decl)
@@ -404,6 +382,7 @@

 (assert (fn? a5))
 (assert (= 'a5 (:name (meta a5))))
+(assert (= *ns* (:ns (meta a5))))
 (assert (= "Assertion a5" (:doc (meta a5))))
 (assert (= ::AssertionPassed (type (a5 1 2))))
 (assert (= ::AssertionFailed (type (a5 2 1))))
@@ -417,6 +396,7 @@
   (assert (= s7 7)))
 (assert (= ::Context (type c7)))
 (assert (= 'c7 (:name (meta c7))))
+(assert (= *ns* (:ns (meta c7))))
 (assert (= "Context c7" (:doc (meta c7))))
 (assert (= [c5] (:parents c7)))
 (assert (fn? (:before c7)))
@@ -430,6 +410,7 @@
   (< a b))

 (assert (= 't10 (:name (meta t10))))
+(assert (= *ns* (:ns (meta t10))))
 (assert (= "Test t10" (:doc (meta t10))))
 (assert (= ::TestCase (type t10)))
 (assert (= [c4 c5] (:contexts t10)))
@@ -441,6 +422,7 @@
 (defsuite t11 "Suite t11" [c4 c5] t9 t10)

 (assert (= 't11 (:name (meta t11))))
+(assert (= *ns* (:ns (meta t11))))
 (assert (= "Suite t11" (:doc (meta t11))))
 (assert (= ::TestCase (type t11)))
 (assert (= [c4 c5] (:contexts t11)))


Updated Fact to version with stubbing

Updated Fact to version with stubbing

--- lib/fact.clj
+++ lib/fact.clj
@@ -71,8 +71,24 @@
           :pending? ~(nil? value-map)
           :params  '~(vec params)
           :values   ~(vec values))))))
-
-(def #^{:doc "The maximum amount of random test values to use."}
+
+(defmacro stub
+  "Create function stubs for isolated unit tests.
+  e.g. (stub [(f 1 2) 3
+              (f 3 2) 5]
+         (= (+ (f 1 2) (f 3 2))
+            8))"
+  [stubs & body]
+  (let [stub-pairs (partition 2 stubs)
+        fold-map   (fn [acc [[f & args] ret]]
+                      (assoc-in acc [f `(list ~@args)] ret))
+        bind-stub  (fn [[fname fhash]]
+                     `(~fname (fn [& args#] (~fhash args#))))]
+    `(binding
+       [~@(mapcat bind-stub (reduce fold-map {} stub-pairs))]
+       ~@body)))
+
+(def #^{:doc "The maximum amount of test values to use per fact."}
   *max-amount* 50)

 (defn- test-cases


add thre manipulation funcs

add thre manipulation funcs

--- sass-dsl/src/sass_dsl/transform.clj
+++ sass-dsl/src/sass_dsl/transform.clj
@@ -75,4 +75,24 @@
    (common/is-selector loc)
    (is-parent-selector loc)))

-
+(defn zip-remove-selector-and-children
+  "Given a zipper location, remove the selector and its children."
+  [loc]
+  (-> loc
+      zip/right
+      zip/remove
+      zip/remove))
+
+(defn pull-selector-and-child-up-one
+  "Given a zipper location - remove it and put it back one level up with a
+  new name."
+  [loc new-node-name]
+  (zip/insert-right
+   (zip/insert-right
+    (if (common/is-first-element loc) ;handle cose where two selectors chained
+      (zip-remove-selector-and-children loc)
+      (zip/up
+       (zip-remove-selector-and chikdren loc)))
+    (zip/node (zip/right loc)))
+   new-node-name))
+


Larger font and center align for single mode

--- src/app/players.cljs
+++ src/app/players.cljs
@@ -26,7 +26,8 @@
              :key (rand-int 1000000)
              :style {:position "absolute"
                      :opacity 1.0
-                     :size 20
-                     :left "40%"
+                     :font-size "28px"
+                     :text-align "center"
+                     :width "90%";
                      :top "30%"}}
        (:text (first @screen) )])))


refactor(swt/display): moved above ui macro

--- ccw.core/src/clj/ccw/swt.clj
+++ ccw.core/src/clj/ccw/swt.clj
@@ -94,6 +94,12 @@
 (import 'org.eclipse.swt.widgets.Display)
 (import 'org.eclipse.swt.widgets.Shell)

+(defn display [] (Display/getCurrent))
+
+(defn active-shell []
+  (-> (org.eclipse.swt.widgets.Display/getDefault)
+    .getActiveShell))
+
 (defmacro ui
   [& body]
   `(if (Display/getCurrent)
@@ -106,12 +112,6 @@
 ;       (fn []
 ;         (let [~display-name (Display/getCurrent)]
 ;           ~@body)))))
-
-(defn display [] (Display/getCurrent))
-
-(defn active-shell []
-  (-> (org.eclipse.swt.widgets.Display/getDefault)
-    .getActiveShell))

 (defn ui-async
   "Executes f asynchronously (non-blocking) on the user-interface thread"


Wrong destructuring. update returns the exit code only

--- jar-async/src/slipstream/async/collector.clj
+++ jar-async/src/slipstream/async/collector.clj
@@ -71,8 +71,8 @@
   [user]
   (let [ch (chan 1)]
     (go
-      (let [[v c] (alts! [ch (timeout timeout-collect)])]
-        (if (nil? v)
+      (let [res (alts! [ch (timeout timeout-collect)])]
+        (if (nil? res)
           (log/log-error
             "Timeout updating metrics for user "
             (.getName user))


Clojure awesomeness makes me high. Added multimethod view. Now you can display both graphs or 3D Objects.

--- src/clj3D/viewer.clj
+++ src/clj3D/viewer.clj
@@ -1,17 +1,17 @@
 (ns clj3D.viewer
   (:gen-class)
   (:use (clj3D fenvs))
-  (:require [incanter.core])
-  (:require [incanter.stats])
+  (:use [incanter core charts stats])
   (import (clj3D.ObjectViewer))
   (:import (java.awt Dimension Toolkit))
   (:import (com.jme3.system AppSettings))
-  (:import (com.jme3.scene Node)))
+  (:import (com.jme3.scene Node Spatial)))

-(defn view
-  "Display on the screen the geometry or node given in input."
-  [object]
+;; Display on the screen the geometry or node given in input.
+;; I have used the defmethod awesomeness to extend the existing
+;; view function, inherited by Incanter
+(defmethod view Spatial [object]
   (let [geometry-node (Node.)
         viewer (clj3D.ObjectViewer.)
         settings (AppSettings. true)


Can't use seqs in logical expressions.

Can't use seqs in logical expressions.

--- src/cljs/go/core.clj
+++ src/cljs/go/core.clj
@@ -241,7 +241,7 @@
 (defn simple-test-expr? [env ast]
   (core/and
     (#{:var :invoke :constant :dot :js} (:op ast))
-    ('#{boolean seq} (cljs.analyzer/infer-tag env ast))))
+    (= 'boolean (cljs.analyzer/infer-tag env ast))))

 (defmacro and
   "Evaluates exprs one at a time, from left to right. If a form


Tweak comment for :selection-delay

Tweak comment for :selection-delay

--- src/clojush/args.clj
+++ src/clojush/args.clj
@@ -439,8 +439,9 @@
           ;; If  this is truthy, then it should be a positive integer d, and all parents
           ;; will be selected with :uniform selection, but also, in each generation for
           ;; which (mod generation d) is 0, before producing offspring, the population
-          ;; will be replaced with the results of repeated selection from an archive of
-          ;; all individuals that have been produced since the previous time this was done.
+          ;; will be replaced with the results of repeated selection (using the specified
+          ;; :parent-selection method) from an archive of all individuals that have been
+          ;; produced since the previous time this was done.

           ;;----------------------------------------
           ;; Arguments related to the Push interpreter


Added commented-out (for now) addition of new uniform addition/combination-and-deletion instructions to :uniform setting for autoconstruction.

--- src/clojush/args.clj
+++ src/clojush/args.clj
@@ -489,6 +489,8 @@
                                         genome_uniform_silence_mutation
                                         genome_uniform_deletion
                                         genome_uniform_addition
+                                        ;genome_uniform_addition_and_deletion
+                                        ;genome_uniform_combination_and_deletion
                                         genome_alternation
                                         genome_uniform_crossover)))]
       (when (not (some #{instr} (:atom-generators @push-argmap)))
@@ -546,3 +548,4 @@

+


Fixed expected values for domain2 after update and added some pauses for the updates to take place before checking for updated values.

--- test/clj/elephantdb/service_test.clj
+++ test/clj/elephantdb/service_test.clj
@@ -330,6 +330,9 @@
           ;; nothing should happen for domain1
           (.update handler "domain1")

+          ;; wait a second
+          (Thread/sleep 1000)
+
           ;; domain1 should not have changed
           (expected-domain-data handler "domain1"
                                 0 [0 0]
@@ -340,12 +343,15 @@
           ;; updating domain2 should cause update and new values being returned
           (.update handler "domain2")

+          ;; wait a second
+          (Thread/sleep 1000)
+
           ;; domain2 should have changed
           (expected-domain-data handler "domain2"
                                 0 [11 11]
-                                1 [21 21]
-                                2 [31 31]
-                                3 [41 41])
+                                1 [22 22]
+                                2 [33 33]
+                                3 [44 44])

           ;; create version 2 for domain1
           (mk-sharded-domain fs dtmp1 domain-spec
@@ -364,6 +370,15 @@
           ;; force update of all domains
           (.updateAll handler)

+          (is (= (thrift/status-loading?
+                  (.getDomainStatus handler "domain1"))))
+          (is (= (thrift/status-loading?
+                  (.getDomainStatus handler "domain2"))))
+
+          ;; wait a second
+          (Thread/sleep 1000)
+
+
           ;; domain1 and domain 2 should have changed
           (expected-domain-data handler "domain1"
                                 0 [1 1]
@@ -377,5 +392,10 @@
                                 2 [34 34]
                                 3 [45 45])

+          (is (= (thrift/status-ready?
+                       (.getDomainStatus handler "domain1"))))
+          (is (= (thrift/status-ready?
+                       (.getDomainStatus handler "domain2"))))
+
           (.shutdown handler))
         ))))


New test: test-render-package-descriptions-special-cases

--- test/clouseau/test_text_renderer.clj
+++ test/clouseau/test_text_renderer.clj
@@ -89,6 +89,12 @@
         (is (= (render-front-page nil "PACKAGE" {"x" "y"} "***" nil nil nil nil) "[Package]\nPACKAGE\n\n[CCS Description]\n***\n\n[x]\ny\n\n"))
         (is (= (render-front-page nil "PACKAGE" {"prod1" "text1" "prod2" "text2"} "***" nil nil nil nil) "[Package]\nPACKAGE\n\n[CCS Description]\n***\n\n[prod1]\ntext1\n\n[prod2]\ntext2\n\n"))))

+(deftest test-render-package-descriptions-special-cases
+    "Check the function clouseau.text-renderer/render-package-descriptions"
+    (testing "the function clouseau.text-renderer/render-package-descriptions"
+        (is (= (render-package-descriptions {"x" nil}) '("[x]\n\n\n")))
+        (is (= (render-package-descriptions {}) '()))))
+
 (deftest test-render-package-descriptions
     "Check the function clouseau.text-renderer/render-package-descriptions"
     (testing "the function clouseau.text-renderer/render-package-descriptions"


Addressed review comments.

Addressed review comments.

--- test/cmr/umm/test/dif/collection.clj
+++ test/cmr/umm/test/dif/collection.clj
@@ -45,9 +45,13 @@
   (for-all [collection coll-gen/dif-collections]
     (let [xml (c/umm->dif-xml collection)
           parsed-dif (c/parse-collection xml)
+          ;; add the required InsertTime and LastUpdate for ECHO10 schema validation
+          data-provider-timestamps (first (gen/sample coll-gen/data-provider-timestamps 1))
+          parsed-dif (assoc parsed-dif :data-provider-timestamps data-provider-timestamps)
           echo10-xml (echo10/umm->echo10-xml parsed-dif)
           parsed-echo10 (echo10-c/parse-collection echo10-xml)]
-      (is (collections-match? parsed-echo10 collection)))))
+      (and (collections-match? parsed-echo10 collection)
+           (= 0 (count (echo10-c/validate-xml echo10-xml)))))))

 ;; This is a made-up include all fields collection xml sample for the parse collection test
 (def all-fields-collection-xml


Import example and example history into separate collections

--- tools/old_import.clj
+++ tools/old_import.clj
@@ -3,7 +3,9 @@
             #_[clojure.java.jdbc.sql :as s]
             [clojure.pprint :refer (pprint)]
             [clojure.string :as str]
-            [somnium.congomongo :as mon]))
+            [somnium.congomongo :as mon]
+            [clojuredocs.data :as data]
+            [clojuredocs.util :as util]))

@@ -24,8 +26,20 @@
 (defn all-notes []
   (j/query mysql-db ["SELECT * FROM comments"]))

+(defn assoc-avatar-url [{:keys [email] :as m}]
+  (if email
+    (assoc m :avatar-url (str "https://www.gravatar.com/avatar/"
+                              (-> email str/lower-case util/md5)
+                              "?r=PG&default=identicon"))
+    m))
+
 (defn lookup-user [user-id]
-  (select-keys (first (j/query mysql-db ["SELECT * FROM users WHERE id=?" user-id])) [:email :login]))
+  (-> (j/query mysql-db ["SELECT * FROM users WHERE id=?" user-id])
+      first
+      (select-keys  [:email :login])
+      (assoc :account-source "clojuredocs")
+      assoc-avatar-url
+      (dissoc :email)))

 (defn lookup-ns [nsid]
   {:ns (->> (j/query mysql-db ["SELECT * FROM namespaces WHERE id=?" nsid])
@@ -44,9 +58,9 @@
        (map (fn [{:keys [updated_at body user_id]}]
               (when user_id ; some example versions don't have a user associated
                 (let [user (lookup-user user_id)]
-                  {:created-at updated_at
+                  {:created-at (.getTime updated_at)
                    :body body
-                   :user user}))))
+                   :editor user}))))
        (filter identity)
        (sort-by :created-at)
        #_reverse))
@@ -58,39 +72,62 @@
   (mon/update! :examples (select-keys e [:library-url :ns :name :body]) e))

 ;; examples
-(time
-  (let [examples (->> (core-nss)
-                      (map (fn [{:keys [library_id name id]}]
-                             {:library-id library_id
-                              :namespace-id id
-                              :namespace name}))
-                      (mapcat (fn [{:keys [namespace namespace-id]}]
-                                (let [res (j/query mysql-db ["SELECT * FROM functions WHERE namespace_id=?" namespace-id])]
-                                  (->> res
-                                       (map (fn [{:keys [name id]}]
-                                              {:ns namespace
-                                               :name name
-                                               :function-id id}))))))
-                      (mapcat (fn [m]
-                                (let [res (j/query mysql-db ["SELECT * FROM examples WHERE function_id=?" (:function-id m)])]
-                                  (->> res
-                                       (map (fn [{:keys [body user_id id created_at updated_at]}]
-                                              (assoc m
-                                                :body body
-                                                :user-id user_id
-                                                :example-id id
-                                                :created-at created_at
-                                                :updated-at updated_at)))))))
-                      (map (fn [{:keys [user-id example-id] :as m}]
-                             (let [user (lookup-user user-id)]
-                               (assoc m
-                                 :user user
-                                 :history (history-for example-id)))))
-                      (map #(assoc % :library-url "https://github.com/clojure/clojure"))
-                      (map #(dissoc % :user-id :example-id :function-id)))]
-    (doseq [e examples]
-      (insert-or-update-example e))
-    (println (mon/fetch-count :examples))))
+
+(do
+  (mon/drop-coll! :examples)
+  (mon/drop-coll! :example-histories)
+  (time
+    (let [examples (->> (core-nss)
+                        (map (fn [{:keys [library_id name id]}]
+                               {:library-id library_id
+                                :namespace-id id
+                                :namespace name}))
+                        (mapcat (fn [{:keys [namespace namespace-id]}]
+                                  (let [res (j/query mysql-db ["SELECT * FROM functions WHERE namespace_id=?" namespace-id])]
+                                    (->> res
+                                         (map (fn [{:keys [name id]}]
+                                                {:ns namespace
+                                                 :name name
+                                                 :function-id id}))))))
+                        (mapcat (fn [m]
+                                  (let [res (j/query mysql-db ["SELECT * FROM examples WHERE function_id=?" (:function-id m)])]
+                                    (->> res
+                                         (map (fn [{:keys [body user_id id created_at updated_at]}]
+                                                (assoc m
+                                                  :body body
+                                                  :user-id user_id
+                                                  :example-id id
+                                                  :created-at created_at
+                                                  :updated-at updated_at)))))))
+                        (map (fn [{:keys [user-id example-id] :as m}]
+                               (let [user (lookup-user user-id)]
+                                 (assoc m
+                                   :author user
+                                   :history (history-for example-id)))))
+                        (map #(assoc % :library-url "https://github.com/clojure/clojure"))
+                        (map (fn [{:keys [author ns name body library-url created-at updated-at history]}]
+                               {:_id (org.bson.types.ObjectId.)
+                                :var {:ns ns
+                                      :name name
+                                      :library-url library-url}
+                                :history history
+                                :author author
+                                :editors (map :editor history)
+                                :body body
+                                :created-at (.getTime created-at)
+                                :updated-at (.getTime updated-at)})))]
+      (doseq [{:keys [_id history] :as e} examples]
+        (mon/insert! :examples
+          (select-keys e [:_id :var :author :editors :body :created-at :updated-at]))
+        (doseq [h history]
+          (mon/insert!
+            :example-histories
+            (merge
+              h
+              {:_id (org.bson.types.ObjectId.)
+               :example-id _id}))))
+      (println "examples" (mon/fetch-count :examples))
+      (println "example history entries" (mon/fetch-count :example-histories)))))

 (def special-forms
   [{:name 'def
@@ -199,25 +236,35 @@

 (def see-alsos (atom {}))

-(time
+#_(time
   (doseq [{:keys [from_id to_id user_id created_at]}
           (j/query mysql-db ["SELECT * FROM see_alsos"])]
     (let [from (pull-ns-name from_id)
           to (pull-ns-name to_id)]
       (swap! see-alsos update-in [from] #(-> %
-                                             (concat [(assoc to :created-at created_at
-                                                             :user (lookup-user user_id))])
+                                             (concat [(-> to
+                                                          (select-keys [:ns :name])
+                                                          (assoc
+                                                              :created-at (.getTime created_at)
+                                                              :author (lookup-user user_id)
+                                                              :library-url "https://github.com/clojure/clojure"
+                                                              )
+                                                          )])
                                              distinct)))))

 (def clj-nss (->> clojure-namespaces
                   (map str)
                   set))

-(time (doseq [[v sas] @see-alsos]
-        (insert-or-update-sa (assoc (select-keys v [:name :ns])
-                               :vars (->> sas
-                                          (filter #(get clj-nss (:ns %))))
-                               :library-url "https://github.com/clojure/clojure"))))
+#_(time
+  (doseq [[v sas] @see-alsos]
+    (data/update-see-also-where!
+      (fn [{:keys [ns name]}] {:var.ns ns :var.name name})
+      {:var (-> v
+                (select-keys [:name :ns])
+                (assoc :library-url "https://github.com/clojure/clojure"))
+       :refs (->> sas
+                  (filter #(get clj-nss (:ns %))))})))

 #_(->> @see-alsos
      (filter #(= "map" (:name (first %))))
@@ -225,9 +272,6 @@
      first
      insert-or-update-var
      pprint)
-
-
-#_(mon/fetch-one :see-alsos :where {:name "fnil"})

@@ -281,7 +325,7 @@
   (mon/update! :namespaces {:name (:name ns-map)} ns-map))

-(let [nss ["clojure.core"
+#_(let [nss ["clojure.core"
            "clojure.data"
            "clojure.edn"
            "clojure.inspector"
@@ -346,19 +390,18 @@
      )

-(defn ins-or-update-note [cmt]
-  (mon/update! :var-notes
-    (select-keys cmt [:user :created-at :var])
-    cmt))
-
 (defn import-notes []
   (let [notes (->> (all-notes)
-                      (map #(assoc % :user (lookup-user (:user_id %))))
-                      (map #(assoc % :var (lookup-function (:commentable_id %))))
-                      (map #(assoc % :created-at (:created_at %)))
-                      (map #(assoc % :updated-at (:updated_at %)))
-                      (map #(select-keys % [:updated-at :var :body :created-at :user])))]
+                   (map #(assoc % :author (lookup-user (:user_id %))))
+                   (map #(assoc % :var (lookup-function (:commentable_id %))))
+                   (map #(assoc % :created-at (:created_at %)))
+                   (map #(assoc % :updated-at (:updated_at %)))
+                   (map (fn [{:keys [created-at updated-at] :as m}]
+                          (assoc m
+                            :created-at (when created-at (.getTime created-at))
+                            :updated-at (when updated-at (.getTime updated-at)))))
+                   (map #(select-keys % [:updated-at :var :body :created-at :author])))]
     (doseq [n notes]
-      (ins-or-update-note n))))
-
-(import-notes)
+      (data/update-note-where! #(select-keys % [:author :body :var]) n))))
+
+#_(import-notes)


[clojure] "Magically" find the app's user namespace which has hooks for starting the app system

--- etc/repl.clj
+++ etc/repl.clj
@@ -70,7 +70,15 @@
   Warning: best if the system is not running, or things will go south

   Example: (R/start-system! 'foo.user)"
-  [an-ns]
+  ([]
+   ;; automagically guess the <app>.user namespace
+   (let [an-ns (-> *ns*
+                   str
+                   (clojure.string/replace #"\..+" ".user")
+                   symbol)]
+     (require an-ns)
+     (start-system! an-ns)))
+  ([an-ns]
   (do
     (printf "!! Starting %s\n" an-ns)
     (if (get @system-status an-ns)
@@ -83,7 +91,7 @@
         (if-let [f (ns-resolve an-ns 'start)]
           (do
             (f)
-            (swap! system-status (fn [s] (assoc s an-ns true)))))))))
+            (swap! system-status (fn [s] (assoc s an-ns true))))))))))

 (defn stop-system!
   "Given a namespace, usually some-service.user, stop the system. If not passed, stops currently running system"


fix expected/actual printing in test results

--- bake/bake/test.clj
+++ bake/bake/test.clj
@@ -74,6 +74,8 @@
     (when-let [message (:message m)] (println message))
     (let [expected (:expected m)
           actual   (:actual m)]
+      (println "expected:" (pr-str expected))
+      (println "  actual:" (pr-str actual))
       (comment when (seq? actual)
         (diff-actual actual)))))


Inbox: Code golf

Inbox: Code golf

--- src-cljs/pepa/workflows/inbox2.cljs
+++ src-cljs/pepa/workflows/inbox2.cljs
@@ -68,30 +68,31 @@
   (will-mount [_]
     (api/fetch-inbox!)))

-(defrecord FakeColumnSource [id]
-  ColumnSource
-  (column-title [_ _]
-    "Fake")
-  (column-pages [_ state]
-    (get-in state [::fake-column-pages]))
-  (remove-pages! [_ state page-ids]
-    (go
-      (js/console.warn "Unimplemented: Removing pages from Fake...")))
-  ColumnDropTarget
-  (accepts-drop? [_ state]
-    true)
-  ;; TODO: Make immutable?
-  (accept-drop! [_ state new-pages]
-    (go
-      (println "dropping" (pr-str new-pages))
-      (om/transact! state ::fake-column-pages
-                    (fn [pages]
-                      (into pages
-                            (remove #(contains? (set (map :id pages)) (:id %)))
-                            new-pages)))
-      (println "Fake-saving...")
-      (<! (async/timeout 2000))
-      (println "Saved!"))))
+(comment
+ (defrecord FakeColumnSource [id]
+   ColumnSource
+   (column-title [_ _]
+     "Fake")
+   (column-pages [_ state]
+     (get-in state [::fake-column-pages]))
+   (remove-pages! [_ state page-ids]
+     (go
+       (js/console.warn "Unimplemented: Removing pages from Fake...")))
+   ColumnDropTarget
+   (accepts-drop? [_ state]
+     true)
+   ;; TODO: Make immutable?
+   (accept-drop! [_ state new-pages]
+     (go
+       (println "dropping" (pr-str new-pages))
+       (om/transact! state ::fake-column-pages
+                     (fn [pages]
+                       (into pages
+                             (remove #(contains? (set (map :id pages)) (:id %)))
+                             new-pages)))
+       (println "Fake-saving...")
+       (<! (async/timeout 2000))
+       (println "Saved!")))))

 (defrecord DocumentColumnSource [id document-id]
   ColumnSource
@@ -239,7 +240,7 @@
   (init-state [_]
     (let [gen (IdGenerator.getInstance)
           columns [(->InboxColumnSource (.getNextUniqueId gen))
-                   (->FakeColumnSource (.getNextUniqueId gen))
+                   #_(->FakeColumnSource (.getNextUniqueId gen))
                    (->DocumentColumnSource (.getNextUniqueId gen) 1)
                    (->DocumentColumnSource (.getNextUniqueId gen) 2)]]
       {:columns (into {}
@@ -258,11 +259,6 @@
     (let [page-cache (make-page-cache state columns)]
       [:.workflow.inbox
        (om/build-all inbox-column columns
-                     {:fn (fn [column]
-                            [state
-                             (-> column
-                                 val
-                                 (assoc ::page-cache page-cache))])
-                      ;; NOTE: Passing this callback as `opts' might
-                      ;; cause issues
+                     {:fn (fn [[_ column]]
+                            [state (assoc column ::page-cache page-cache)])
                       :state {:handle-drop! (partial inbox-handle-drop! state owner page-cache)}})])))


[#8] Get the .p12 file and store it temporary

--- api/src/api/core.clj
+++ api/src/api/core.clj
@@ -7,7 +7,20 @@
            [com.google.apphosting.utils.config AppEngineWebXmlReader]
            [java.io ByteArrayInputStream])
   (:require [clojure.string :as str]
+            [clojure.java.io :as io]
             [clj-http.client :as http]))
+
+(def p12-dir (System/getProperty "java.io.tmpdir"))
+
+#_(let [auth-token ""
+        email ""
+        alias-map (get-alias-map instance-map)
+        instance-id (get alias-map "uat1")
+        host (str instance-id ".appspot.com")
+        port 443
+        iam-account (get-in instance-map [instance-id "serviceAccountId"])
+        p12-file (get-p12 instance-id auth-token)]
+    (get-filtered-surveys host iam-account (.getAbsolutePath p12-file) email))

 (defn get-filtered-surveys
   [host iam-account p12-path email]
@@ -53,7 +66,8 @@
                       :content))))
         ae-reader (AppEngineWebXmlReader. (System/getProperty "java.io.tmpdir")
                                           (.getName temp-file))]
-    (.getSystemProperties (.readAppEngineWebXml ae-reader))))
+    {:instance-id instance-id
+     :properties (.getSystemProperties (.readAppEngineWebXml ae-reader))}))

 (defn get-instance-map [auth-token]
   (let [instances (get-instances auth-token)]
@@ -67,3 +81,38 @@
                      (get-instance-props instance-id auth-token)
                      (catch clojure.lang.ExceptionInfo e)))
                  instances))))
+
+(defn get-alias-map [instances-map]
+  (reduce (fn [m [instance-id props]]
+            (let [url (get props "alias")
+                  alias (first (str/split url #"\."))]
+              (assoc m alias instance-id)))
+          {}
+          instances-map))
+
+
+
+(defn fetch-p12
+  "Fetch the p12 file from github. Save it and return the File.
+  Returns nil if the file can't be found"
+  [instance-id auth-token]
+  (try
+    (let [bytes (-> (http/get (format
+                               "https://api.github.com/repos/akvo/akvo-flow-server-config/contents/%1$s/%1$s.p12"
+                               instance-id)
+                              {:headers {"Authorization" (str "token " auth-token)
+                                         "Content-Type" "application/json"}
+                               :as :json})
+                    :body
+                    :content
+                    Base64/decodeBase64)
+          file (io/file p12-dir (str instance-id ".p12"))]
+      (io/copy bytes file)
+      file)
+    (catch clojure.lang.ExceptionInfo e)))
+
+(defn get-p12 [instance-id auth-token]
+  (let [file (io/file p12-dir (str instance-id ".p12"))]
+    (if (.exists file)
+      file
+      (fetch-p12 instance-id auth-token))))


Directly eval CljRbUtil rather than adding to load path

--- src/clj/rb.clj
+++ src/clj/rb.clj
@@ -46,7 +46,7 @@
       method-name (object-array (map #(clj->rb % rt) args)))))

 (defn- rb-helper [rt]
-  (eval rt "require 'clj_rb_util';CljRbUtil"))
+  (eval rt "CljRbUtil"))

 (defn- ruby-runtime [rt]
   (-> rt .getProvider .getRuntime))
@@ -141,13 +141,13 @@
      (let [rt (ScriptingContainer. (if preserve-locals?
                                      LocalVariableBehavior/PERSISTENT
                                      LocalVariableBehavior/TRANSIENT))]
-       (.setLoadPaths rt (conj load-paths
-                           (.toExternalForm (io/resource "clj-ruby-helpers"))))
+       (.setLoadPaths rt load-paths)
        (when-let [paths (seq gem-paths)]
          (setenv rt "GEM_PATH" (str/join ":" (map pr-str paths))))
        (doseq [[k v] env]
          (setenv rt k v))
-       (require "rubygems")
+       (eval-file rt (-> "clj-ruby-helpers/clj_rb_util.rb" io/resource io/reader))
+       (require rt "rubygems")
        rt)))


removing core.async dependencies from defview and devquery

--- libs/axiom-cljs/src/axiom_cljs/macros.clj
+++ libs/axiom-cljs/src/axiom_cljs/macros.clj
@@ -1,5 +1,4 @@
-(ns axiom-cljs.macros
-  (:require [cljs.core.async.macros :refer [go go-loop]]))
+(ns axiom-cljs.macros)

 (defn ^:private symbols [form]
@@ -37,8 +36,7 @@
                          writers #{'$user}}}]
   (let [fact-name (-> fact first str (subs 1))]
     `(defonce ~name
-       (let [sub-chan# (async/chan 1)
-             state# ~store-in]
+       (let [state# ~store-in]
          (reset! state# {})
          ((:sub ~host) ~fact-name
           (fn [ev#]
@@ -100,7 +98,6 @@
         pred-name (-> pred-name str (subs 1))]
     `(defonce ~name
        (let [id-map# (atom {})
-             sub-chan# (async/chan 1)
              state# ~store-in]
          (reset! ~store-in {})
          ((:sub ~host) ~(str pred-name "!")


type hinting and multiple arity

type hinting and multiple arity

--- src/ganjika/core.clj
+++ src/ganjika/core.clj
@@ -9,43 +9,69 @@
      #(str %1 (uppercase-to-hyphened-lowercase %2))
      "" name)))

-(defn- method-spec [m]
-  {:name (camel-to-kebab-case (.getName m))
-   :raw-name (.getName m)
-   :args (.getParameterCount m)
-   :is-void (-> m
-                 .getReturnType
-                 .getName
-                 (= "void"))
-   :is-static (java.lang.reflect.Modifier/isStatic (.getModifiers m))})
+(defn- is-public [^java.lang.reflect.Method m]
+  (java.lang.reflect.Modifier/isPublic (.getModifiers m)))

-(defn- declare-fn
-  "Given a method spec create an unevaluated function form, currying
-  over the provided instance"
-  [spec instance no-currying]
-  (let [method-name (symbol (:name spec))
-        raw-method-name (symbol (str "." (:raw-name spec)))
-        static-method-name (symbol (str (.getName instance) "/" (:raw-name spec)))
-        params (vec (map gensym (range (:args spec))))
+(defn- method-specs [^java.lang.Class c]
+  (->> c
+       .getDeclaredMethods
+       (filter is-public)
+       (map (fn [m]
+              {:class-name (.getName c)
+               :name (camel-to-kebab-case (.getName m))
+               :raw-name (.getName m)
+               :param-count (.getParameterCount m)
+               :is-void (-> m
+                            .getReturnType
+                            .getName
+                            (= "void"))
+               :param-types (.getParameterTypes m)
+               :is-static (java.lang.reflect.Modifier/isStatic (.getModifiers m))}))))
+
+(defn hint-params [spec]
+  (let [boxer {"int" "java.lang.Integer" "long" "java.lang.Long"
+               "short" "java.lang.Short" "float" "java.lang.Float"
+               "double" "java.lang.Double" "boolean" "java.lang.Boolean"
+               "byte" "java.lang.Byte" "char" "java.lang.Character"}]
+    (map #(let [name (.getName %)]
+            (str "^" (or (boxer name) name)))
+         (:param-types spec))))
+
+(defn- build-arity
+  "TODO"
+  [instance no-currying use-type-hinting spec]
+  (let [raw-method-name (symbol (str "." (:raw-name spec)))
+        static-method-name (when (:is-static spec) (symbol (str (:class-name spec) "/" (:raw-name spec))))
+        params (vec (map gensym (range (:param-count spec))))
+        hinted-params (if use-type-hinting (interleave (hint-params spec) params) params)
         is-void (:is-void spec)]
     (cond
      ;; define function for static method
      (:is-static spec)
-     `(def ~method-name (fn ([~@params] (~static-method-name ~@params))))
+     `([~@params] (~static-method-name ~@params))
      ;; define non-curried function
      no-currying
-     `(defn ~method-name [this# ~@params]
-        (let [return# (~raw-method-name this# ~@params)]
-          (if ~is-void
-            this#
-            return#)))
+     `([this# ~@params]
+       (let [result# (~raw-method-name this# ~@params)]
+         (if ~is-void
+           this#
+           result#)))
      ;; define curried function
      :else
-     `(defn ~method-name [~@params]
-        (let [return# (~raw-method-name @~instance ~@params)]
-          (if ~is-void
-            @~instance
-            return#))))))
+     `([~@params]
+       (let [result# (~raw-method-name @~instance ~@params)]
+         (if ~is-void
+           @~instance
+           result#))))))
+
+(defn build-function
+   ;; TODO overloaded arities
+  "TODO make private"
+  [[fn-name specs] target-sym no-currying type-hinting]
+  (let [fn-symbol (symbol fn-name)
+        use-type-hinting type-hinting
+        arity-fn (partial build-arity target-sym no-currying use-type-hinting)]
+    `(defn ~fn-symbol ~@(map arity-fn specs))))

 (defn- define-symbol
   "Creates and return a symbol that resolves to the provided object"
@@ -60,29 +86,28 @@
   namespace). If target is a class an instance is created (assumes
   default constructor) unless currying is disabled (via :currying
   false)."
-  [target & {:keys [using-ns currying]}]
+  [target & {:keys [using-ns currying disable-type-hinting]}]
   (let [no-currying (false? currying)
+        type-hinting (not (true? disable-type-hinting))
         target (eval target)
         [clazz instance] (cond
                           (and (class? target) no-currying) [target nil]
                           no-currying [(class target) nil]
                           (class? target) [target (.newInstance target)]
                           :else [(class target) target])
-        specs (map method-spec (.getDeclaredMethods clazz))
+        specs (method-specs clazz)
+        specs-by-name (group-by :name specs)
         target-sym (if no-currying clazz (define-symbol instance))
         current-ns (.getName *ns*)]
     (when-not no-currying (assert (instance? clazz instance)))
     `(do
        (when ~using-ns
          (in-ns ~using-ns))
-       ~@(map #(declare-fn % target-sym no-currying) specs)
+       ~@(map #(build-function % target-sym no-currying type-hinting) specs-by-name)
        (in-ns (quote ~current-ns)))))

 (comment
-  ;;- multiple arity
-  ;;- currying must be disabled by default for static methods
-  ;; and field/static methods with the same name??? renaming?
-  ;;- and only public methods... or make them public????
+  ;;- field/static methods with the same name??? renaming?
   ;;- better interop by coercing argument types
   (refer 'fus.noma.da :only ['mouse-move])
   (to-clj java.awt.Robot :using-ns 'fus.noma.da))


added some default logic

added some default logic

--- src/k_means/core.clj
+++ src/k_means/core.clj
@@ -17,10 +17,15 @@
 (defn mean
   "Retrieve the mean value of a list"
   [l]
-  (/ (reduce + l) (count l)))
+  (float (/ (reduce + l) (count l))))

-(defn mean-vector-centroid
+(defn mean-centroids
   "Calculates the mean for a given centroid"
+  [c l]
+  c)
+
+(defn cluster
+  "take a vector and cluster it via distance"
   [c l]
   c)
 ; (defn centroid)
@@ -42,7 +47,7 @@
     (cond
       (empty? l) g
       :else
-        (recur (rest l) c ())))
+        (recur (rest l) c ()))))

 (defn k-means
   "Perform the actual k-means"
@@ -50,11 +55,12 @@
     (k-means l k (choose-initial-centroids l k) (make-array Float/TYPE k)))
   ([l k c g]
     (cond
+      ; These functions aren't actually correct
       (< (count g) (count l)) ; Add in new data to the clusters
-        (recur (rest l) k (mean-centroids c) (cluster g)))
+        (recur (rest l) k (mean-centroids c g) (cluster g l))
       (still-moving? c) ; Keep shifting centroids until equilibrium
-        (recur nil k (mean-centroids c) (cluster g))
-      :else g))
+        (recur nil k (mean-centroids c g) (cluster g l))
+      :else g)))

 (defn -main
   "Main function for k-means"


add ability to issue query against books on a shelf

--- src/myshelf/core.clj
+++ src/myshelf/core.clj
@@ -133,15 +133,17 @@
 (defn get-books-on-shelf
   "Returns list of hashmaps representing the books on a given
   user's bookshelf"
-  [consumer access-token user-id shelf]
+  [consumer access-token user-id shelf & [query]]
   (let [shelf-url (str "https://www.goodreads.com/review/list/"
                        user-id)
+        params (merge {:shelf shelf :format "xml" :v 2}
+                      (when query
+                        {(keyword "search[query]")
+                         query}))
         resp (make-auth-request-GET consumer
                                     access-token
                                     shelf-url
-                                    {:shelf shelf
-                                     :format "xml"
-                                     :v 2})]
+                                    params)]
     (->> resp
          :content
          second