(ns iwaswhere-web.store
  "This namespace contains the functions necessary to instantiate the store-cmp,
   which then holds the server side application state."
  (:require [iwaswhere-web.files :as f]
            [taoensso.timbre.profiling :refer [p profile]]
            [iwaswhere-web.graph.query :as gq]
            [iwaswhere-web.graph.stats :as gs]
            [iwaswhere-web.graph.add :as ga]
            [iwaswhere-web.specs]
            [ubergraph.core :as uber]
            [iwaswhere-web.keepalive :as ka]
            [clojure.tools.logging :as log]
            [me.raynes.fs :as fs]
            [iwaswhere-web.fulltext-search :as ft]
            [clojure.pprint :as pp]
            [clojure.edn :as edn]
            [clojure.java.io :as io]))

(defn read-dir
  [state entries-to-index cfg custom-path]
  (let [path (:daily-logs-path (f/paths cfg custom-path))
        files (file-seq (clojure.java.io/file path))]
    (doseq [f (f/filter-by-name files #"\d{4}-\d{2}-\d{2}.jrn")]
      (with-open [reader (clojure.java.io/reader f)]
        (let [lines (line-seq reader)]
          (doseq [line lines]
            (try
              (let [parsed (clojure.edn/read-string line)
                    ts (:timestamp parsed)]
                (if (:deleted parsed)
                  (do (swap! state ga/remove-node ts)
                      (swap! entries-to-index dissoc ts))
                  (do (swap! entries-to-index assoc-in [ts] parsed)
                      (swap! state ga/add-node ts parsed :startup))))
              (catch Exception ex
                (log/error "Exception" ex "when parsing line:\n" line)))))))))

(defn load-cfg
  "Load config from file. When not exists, use default config and write the
   default to data path."
  []
  (let [conf-path (str f/data-path "/conf.edn")
        default (edn/read-string (slurp (io/resource "default-conf.edn")))]
    (try (edn/read-string (slurp conf-path))
         (catch Exception ex
           (do (log/warn "No config found -> copying from default.")
               (spit conf-path (with-out-str (pp/pprint default)))
               default)))))

(defn state-fn
  "Initial state function, creates state atom and then parses all files in
   data directory into the component state.
   Entries are stored as attributes of graph nodes, where the node itself is
   timestamp of an entry. A sort order by descending timestamp is maintained
   in a sorted set of the nodes."
  [put-fn]
  (let [conf (load-cfg)
        entries-to-index (atom {})
        state (atom {:sorted-entries (sorted-set-by >)
                     :graph          (uber/graph)
                     :cfg            conf
                     :lucene-index   ft/index
                     :client-queries {}
                     :hashtags       #{}
                     :mentions       #{}
                     :stats          {:entry-count     0
                                      :node-count      0
                                      :edge-count      0
                                      :daily-summaries {}}})]
    (read-dir state entries-to-index conf nil)

    (future
      (log/info "Summary stats creation started")
      (let [days-to-summarize (:days-to-summarize @state)
            t (with-out-str
                (time (doseq [[day-node snapshot] days-to-summarize]
                        (swap! state gs/mk-daily-summary snapshot day-node))))
            cnt (count days-to-summarize)]
        (swap! state dissoc :days-to-summarize)
        (log/info "Created summary stats for" cnt "days." t)))

    (future
      (Thread/sleep 2000)
      (log/info "Indexing started")
      (let [t (with-out-str
                (time (doseq [entry (vals @entries-to-index)]
                        (put-fn [:ft/add entry]))))]
        (log/info "Indexed" (count @entries-to-index) "entries." t))
      (reset! entries-to-index []))
    {:state state}))

(defn refresh-cfg
  "Refresh configuration by reloading the config file."
  [{:keys [current-state]}]
  {:new-state (assoc-in current-state [:cfg] (load-cfg))})

(defn cmp-map
  "Generates component map for state-cmp."
  [cmp-id]
  {:cmp-id      cmp-id
   :state-fn    state-fn
   :opts        {:msgs-on-firehose true}
   :handler-map (merge
                  gs/stats-handler-map
                  {:entry/import   f/entry-import-fn
                   :entry/find     gq/find-entry
                   :entry/update   f/geo-entry-persist-fn
                   :entry/trash    f/trash-entry-fn
                   :state/search   gq/query-fn
                   :cfg/refresh    refresh-cfg
                   :cmd/keep-alive ka/keepalive-fn})})

(ns iwaswhere-web.store
  "This namespace contains the functions necessary to instantiate the store-cmp,
   which then holds the server side application state."
  (:require [iwaswhere-web.files :as f]
            [taoensso.timbre.profiling :refer [p profile]]
            [iwaswhere-web.graph.query :as gq]
            [iwaswhere-web.graph.stats :as gs]
            [iwaswhere-web.graph.add :as ga]
            [iwaswhere-web.specs]
            [ubergraph.core :as uber]
            [iwaswhere-web.keepalive :as ka]
            [clojure.tools.logging :as log]
            [me.raynes.fs :as fs]
            [iwaswhere-web.fulltext-search :as ft]
            [clojure.pprint :as pp]
            [clojure.edn :as edn]
            [clojure.java.io :as io]))

(defn read-dir
  [state entries-to-index cfg custom-path]
  (let [path (:daily-logs-path (f/paths cfg custom-path))
        files (file-seq (clojure.java.io/file path))]
    (doseq [f (f/filter-by-name files #"\d{4}-\d{2}-\d{2}.jrn")]
      (with-open [reader (clojure.java.io/reader f)]
        (let [lines (line-seq reader)]
          (doseq [line lines]
            (try
              (let [parsed (clojure.edn/read-string line)
                    ts (:timestamp parsed)]
                (if (:deleted parsed)
                  (do (swap! state ga/remove-node ts)
                      (swap! entries-to-index dissoc ts))
                  (do (swap! entries-to-index assoc-in [ts] parsed)
                      (swap! state ga/add-node ts parsed :startup))))
              (catch Exception ex
                (log/error "Exception" ex "when parsing line:\n" line)))))))))

(defn load-cfg
  "Load config from file. When not exists, use default config and write the
   default to data path."
  []
  (let [conf-path (str f/data-path "/conf.edn")
        default (edn/read-string (slurp (io/resource "default-conf.edn")))]
    (try (edn/read-string (slurp conf-path))
         (catch Exception ex
           (do (log/warn "No config found -> copying from default.")
               (fs/mkdirs f/data-path)
               (spit conf-path (with-out-str (pp/pprint default)))
               default)))))

(defn state-fn
  "Initial state function, creates state atom and then parses all files in
   data directory into the component state.
   Entries are stored as attributes of graph nodes, where the node itself is
   timestamp of an entry. A sort order by descending timestamp is maintained
   in a sorted set of the nodes."
  [put-fn]
  (let [conf (load-cfg)
        entries-to-index (atom {})
        state (atom {:sorted-entries (sorted-set-by >)
                     :graph          (uber/graph)
                     :cfg            conf
                     :lucene-index   ft/index
                     :client-queries {}
                     :hashtags       #{}
                     :mentions       #{}
                     :stats          {:entry-count     0
                                      :node-count      0
                                      :edge-count      0
                                      :daily-summaries {}}})]
    (read-dir state entries-to-index conf nil)

    (future
      (log/info "Summary stats creation started")
      (let [days-to-summarize (:days-to-summarize @state)
            t (with-out-str
                (time (doseq [[day-node snapshot] days-to-summarize]
                        (swap! state gs/mk-daily-summary snapshot day-node))))
            cnt (count days-to-summarize)]
        (swap! state dissoc :days-to-summarize)
        (log/info "Created summary stats for" cnt "days." t)))

    (future
      (Thread/sleep 2000)
      (log/info "Indexing started")
      (let [t (with-out-str
                (time (doseq [entry (vals @entries-to-index)]
                        (put-fn [:ft/add entry]))))]
        (log/info "Indexed" (count @entries-to-index) "entries." t))
      (reset! entries-to-index []))
    {:state state}))

(defn refresh-cfg
  "Refresh configuration by reloading the config file."
  [{:keys [current-state]}]
  {:new-state (assoc-in current-state [:cfg] (load-cfg))})

(defn cmp-map
  "Generates component map for state-cmp."
  [cmp-id]
  {:cmp-id      cmp-id
   :state-fn    state-fn
   :opts        {:msgs-on-firehose true}
   :handler-map (merge
                  gs/stats-handler-map
                  {:entry/import   f/entry-import-fn
                   :entry/find     gq/find-entry
                   :entry/update   f/geo-entry-persist-fn
                   :entry/trash    f/trash-entry-fn
                   :state/search   gq/query-fn
                   :cfg/refresh    refresh-cfg
                   :cmd/keep-alive ka/keepalive-fn})})

create config when not exists

create config when not exists


(ns searchbot.widgets
  (:require-macros [cljs.core.async.macros :refer [go alt!]])
  (:require [om.core :as om :include-macros true]
            [om-tools.core :refer-macros [defcomponent]]
            [sablono.core :as html :refer-macros [html]]
            [cljs-http.client :as http]
            [cljs.core.async :refer [put! <! >! chan timeout]]
            [searchbot.charts :refer [es-chart]]
            ))

(defn es-url [purpose]
  (case purpose
    :count "/es/stats_avc_30d/avc/_count"
    :agg "/es/stats_avc_1d/avc/_search"
    "/es"))

(def poll-interval 30000)

(defn commify [s]
  (let [s (reverse s)]
    (loop [[group remainder] (split-at 3 s)
           result '()]
      (if (empty? remainder)
        (apply str (reverse (rest (concat result (list \,) group)))) ; rest to remove extraneous ,
        (recur (split-at 3 remainder) (concat result (list \,) group))))))

(defn es-count []
  (let [c (chan)]
    (go (let [{{docCount :count} :body} (<! (http/get (es-url :count)))]
          (>! c docCount)))
    c))

(defn es-agg [url post-body]
  (let [c (chan)]
    (go (let [{agg :body} (<! (http/post url {:json-params
                                              (-> post-body
                                                  (assoc :search_type "count")
;;                                                   (assoc :preference "_primary")
                                                  )}))]
          (>! c agg)))
    c))

(defcomponent header [app owner opts]
  (will-mount [_]
              (go (while true
                    (let [docCount (<! (es-count))]
                      (om/update! app [:es-count] docCount))
                    (<! (timeout poll-interval)))))
  (render [_]
          (html [:h3 (:header-text app) " with " (-> app :es-count str commify) " raw data in ES"])))

(defcomponent agg-summary [app owner opts]
  (render [this] (html [:.col.s4
                        [:.card.blue-grey.darken-1
                         [:.card-content.amber-text.text-accent-4
                          [:table.responsive-table.hoverable.bordered ;.table.table-hover.table-condensed
                           [:thead
                            [:tr
                             [:th {:align "right"} "Aggregation"]
                             [:th {:align "right"} "Records aggregated"]
                             [:th {:align "right"} "Time (ms)"]]]
                           [:tbody
                            (for [aggKey (filter #(not= :div %) (-> app :agg keys))]
                              [:tr
                               [:td [:span (name aggKey)]]
                               [:td (-> app :agg aggKey :hits :total str commify)]
                               [:td (-> app :agg aggKey :took str commify)]])]]]]])))

(defcomponent detail [app owner opts]
  (render [_]
          (html
           [:pre {:style {:font-size "8pt"}}
            [:code.json (.stringify js/JSON (clj->js opts) nil 4)]]
           ))
  )

(defcomponent agg-table [app owner opts]
  (render [this] (html [:.card.blue-grey.darken-1
                        [:.card-content.activator.waves-effect.waves-block.waves-light.amber-text.text-accent-4
                         [:span.card-title (-> opts :agg-key name)
                          [:a.btn-floating.btn-flat.waves-effect.waves-light.activator.right
                           [:i.mdi-action-settings.right]]]
                         [:table.responsive-table.hoverable.bordered ;.table.table-hover.table-condensed
                          [:thead
                           [:tr
                            (for [col (:header opts)]
                              [:th (:label col)])]]
                          [:tbody
                           (let [agg-key (-> opts :agg-key keyword)
                                 agg-top (-> opts :agg-top keyword)]
                             (for [x (-> app :agg (get agg-key) :aggregations agg-top :buckets)]
                               [:tr
                                [:td (->> opts :header first :agg keyword (get x))]
                                (for [d (->> opts :header rest)]
                                  [:td (-> x (get-in [(keyword (:agg d)) :value]) str commify)])]))]]]
                        [:.card-reveal
                         [:span.card-title.grey-text.text-darken-4
                          (:agg-key opts)
                          [:i.mdi-navigation-close.right]
                          [:ul.collapsible.popout {:data-collapsible "accordion"}
                           [:li
                            [:div.collapsible-header [:i.fa.fa-trello] "Spec"]
                            [:div.collapsible-body
                             (om/build detail app {:opts opts})
                             ]]
                           [:li
                            [:div.collapsible-header [:i.fa.fa-list-ol] "Data"]
                            [:div.collapsible-body
                             (let [agg-key (-> opts :agg-key keyword)
                                   agg-top (-> opts :agg-top keyword)
                                   table-data (-> app :agg (get agg-key) :aggregations agg-top)]
;;                                (.log js/console (pr-str table-data))
                               (om/build detail app {:opts table-data}))
                             ]]
                           ]
                         ]]])))

(defcomponent aggregator [app owner opts]
  (will-mount [_]
              (go (while true
                    (let [agg-key (-> opts :agg-key keyword)
                          agg-resp (<! (es-agg (or (:url opts) (es-url :agg))
                                               (merge (:body opts)
                                                       {:query
                                                        {:filtered
                                                         {:query {:match_all {}}
                                                          :filter {:range {:_timestamp {:gte (or (:gte opts) "now-24h") :lte "now"}}}}}})))]
;;                       (.log js/console (pr-str agg-key))
;;                       (.log js/console (pr-str agg-resp))
                      (om/update! app [:agg agg-key] agg-resp))
                    (<! (timeout poll-interval)))))
  (render [_] (html [:.hide "Aggregator:" (-> opts :agg-key)])))

;;;;;;;;;;;;;;;;;;
;; meta components

(defn fetch-meta [url]
  (let [c (chan)]
    (go (let [{body :body} (<! (http/get url))]
          (>! c body)))
    c))

(defcomponent aggregators [app owner opts]
;;   (will-mount [_]
;;               (go (while true
;;                     (let [{_aggs :aggregators} (<! (fetch-meta "/_aggregators"))]
;;                       (om/update! app [:aggregators] _aggs))
;;                     (<! (timeout poll-interval)))))
  (render [_] (html [:div (for [agg (:aggregators app)] (om/build aggregator app {:opts agg}))])))

(defn- get-component
  [widget-type]
  (case widget-type
    "es-chart" es-chart
    "agg-table" agg-table
    "agg-summary" agg-summary
    "header" header
    nil))

(defn- build-component
  [app widget]
  (let [component (get-component (:type widget))
        cursor (-> widget :cursor keyword)]
    (if component (om/build component (if cursor (get app cursor) app) {:opts widget}))))

(defn- col-class
  [count-per-row]
  (case count-per-row
    1 "s12"
    2 "s6"
    3 "s4"
    4 "s3"
    "s3"))

(defn- build-row [app row]
  [:.row (for [widget row]
           [:div {:class (str "col " (col-class (count row)))}
            (build-component app widget)]
           )])

(defcomponent widgets [app owner opts]
;;   (will-mount [_]
;;               (go (while true
;;                     (let [{_widgets :widgets} (<! (fetch-meta "/_widgets"))]
;;                       (om/update! app [:widgets] _widgets))
;;                     (<! (timeout poll-interval)))))
  (render [_] (html [:div (for [row (:widgets app)] (build-row app row))])))

;;;;;;;;;;;;;;;;

(defn init-app-state
  [state]
  (go (let [from-server (<! (fetch-meta "/_init"))]
        (om/update! state from-server)
        )))

;;;;;;;;;;;;;;;;

(defcomponent counter [data :- {:init js/Number} owner]
  (will-mount [_]
              (om/set-state! owner :n (:init data)))
  (render-state [_ {:keys [n]}]
                (html [:div
                       [:span (str "Count: " n)]
                       [:button
                        {:on-click #(om/set-state! owner :n (inc n))}
                        "+"]
                       [:button
                        {:on-click #(om/set-state! owner :n (dec n))}
                        "-"]])))

(ns searchbot.widgets
  (:require-macros [cljs.core.async.macros :refer [go alt!]])
  (:require [om.core :as om :include-macros true]
            [om-tools.core :refer-macros [defcomponent]]
            [sablono.core :as html :refer-macros [html]]
            [cljs-http.client :as http]
            [cljs.core.async :refer [put! <! >! chan timeout]]
            [searchbot.charts :refer [es-chart]]
            ))

(def poll-interval 30000)

(defn commify [s]
  (let [s (reverse s)]
    (loop [[group remainder] (split-at 3 s)
           result '()]
      (if (empty? remainder)
        (apply str (reverse (rest (concat result (list \,) group)))) ; rest to remove extraneous ,
        (recur (split-at 3 remainder) (concat result (list \,) group))))))

(defn es-count [api-url]
  (let [c (chan)]
    (go (let [{{docCount :count} :body} (<! (http/get api-url))]
          (>! c docCount)))
    c))

(defn es-agg [api-url post-body]
  (let [c (chan)]
    (go (let [{agg :body} (<! (http/post api-url
                                         {:json-params
                                          (-> post-body (assoc :search_type "count"))}))]
          (>! c agg)))
    c))

(defcomponent header [app owner opts]
  (will-mount [_]
              (go (while true
                    (let [docCount (<! (es-count (-> app :es-api :count)))]
                      (om/update! app [:es-count] docCount))
                    (<! (timeout (or (-> app :poll-interval) poll-interval))))))
  (render [_]
          (html [:h3 (:header-text app) " with " (-> app :es-count str commify) " raw data in ES"])))

(defcomponent agg-summary [app owner opts]
  (render [this] (html [:.col.s4
                        [:.card.blue-grey.darken-1
                         [:.card-content.amber-text.text-accent-4
                          [:table.responsive-table.hoverable.bordered ;.table.table-hover.table-condensed
                           [:thead
                            [:tr
                             [:th {:align "right"} "Aggregation"]
                             [:th {:align "right"} "Records aggregated"]
                             [:th {:align "right"} "Time (ms)"]]]
                           [:tbody
                            (for [aggKey (filter #(not= :div %) (-> app :agg keys))]
                              [:tr
                               [:td [:span (name aggKey)]]
                               [:td (-> app :agg aggKey :hits :total str commify)]
                               [:td (-> app :agg aggKey :took str commify)]])]]]]])))

(defcomponent detail [app owner opts]
  (render [_]
          (html
           [:pre {:style {:font-size "8pt"}}
            [:code.json (.stringify js/JSON (clj->js opts) nil 4)]])))

(defcomponent agg-table [app owner opts]
  (render [this] (html [:.card.blue-grey.darken-1
                        [:.card-content.activator.waves-effect.waves-block.waves-light.amber-text.text-accent-4
                         [:span.card-title (-> opts :agg-key name)
                          [:a.btn-floating.btn-flat.waves-effect.waves-light.activator.right
                           [:i.mdi-action-settings.right]]]
                         [:table.responsive-table.hoverable.bordered ;.table.table-hover.table-condensed
                          [:thead
                           [:tr
                            (for [col (:header opts)]
                              [:th (:label col)])]]
                          [:tbody
                           (let [agg-key (-> opts :agg-key keyword)
                                 agg-top (-> opts :agg-top keyword)]
                             (for [x (-> app :agg (get agg-key) :aggregations agg-top :buckets)]
                               [:tr
                                [:td (->> opts :header first :agg keyword (get x))]
                                (for [d (->> opts :header rest)]
                                  [:td (-> x (get-in [(keyword (:agg d)) :value]) str commify)])]))]]]
                        [:.card-reveal
                         [:span.card-title.grey-text.text-darken-4
                          (:agg-key opts)
                          [:i.mdi-navigation-close.right]
                          [:ul.collapsible.popout {:data-collapsible "accordion"}
                           [:li
                            [:div.collapsible-header [:i.fa.fa-trello] "Spec"]
                            [:div.collapsible-body
                             (om/build detail app {:opts opts})
                             ]]
                           [:li
                            [:div.collapsible-header [:i.fa.fa-list-ol] "Data"]
                            [:div.collapsible-body
                             (let [agg-key (-> opts :agg-key keyword)
                                   agg-top (-> opts :agg-top keyword)
                                   table-data (-> app :agg (get agg-key) :aggregations agg-top)]
;;                                (.log js/console (pr-str table-data))
                               (om/build detail app {:opts table-data}))
                             ]]
                           ]
                         ]]])))

(defcomponent aggregator [app owner opts]
  (will-mount [_]
              (go (while true
                    (let [agg-key (-> opts :agg-key keyword)
                          agg-resp (<! (es-agg (or (:url opts) (-> app :es-api :agg))
                                               (merge (:body opts)
                                                       {:query
                                                        {:filtered
                                                         {:query {:match_all {}}
                                                          :filter {:range {:_timestamp {:gte (or (:gte opts) "now-24h") :lte "now"}}}}}})))]
;;                       (.log js/console (pr-str agg-key))
;;                       (.log js/console (pr-str agg-resp))
                      (om/update! app [:agg agg-key] agg-resp))
                    (<! (timeout (or (-> app :poll-interval) poll-interval))))))
  (render [_] (html [:.hide "Aggregator:" (-> opts :agg-key)])))

;;;;;;;;;;;;;;;;;;
;; meta components

(defcomponent aggregators [app owner opts]
  (render [_] (html [:div (for [agg (:aggregators app)]
                            (om/build aggregator app {:opts agg}))])))

(defn- get-component
  [widget-type]
  (case widget-type
    "es-chart" es-chart
    "agg-table" agg-table
    "agg-summary" agg-summary
    "header" header
    nil))

(defn- build-component
  [app widget]
  (let [component (get-component (:type widget))
        cursor (-> widget :cursor keyword)]
    (if component (om/build component (if cursor (get app cursor) app) {:opts widget}))))

(defn- col-class
  [count-per-row]
  (case count-per-row
    1 "s12"
    2 "s6"
    3 "s4"
    4 "s3"
    "s3"))

(defn- build-row [app row]
  [:.row (for [widget row]
           [:div {:class (str "col " (col-class (count row)))}
            (build-component app widget)]
           )])

(defcomponent widgets [app owner opts]
  (render [_] (html [:div (for [row (:widgets app)] (build-row app row))])))

;;;;;;;;;;;;;;;;

(defn fetch-meta [url]
  (let [c (chan)]
    (go (let [{body :body} (<! (http/get url))]
          (>! c body)))
    c))

(defn init-app-state
  [state]
  (go (let [from-server (<! (fetch-meta "/_init"))]
        (om/update! state from-server)
        )))

;;;;;;;;;;;;;;;;

(defcomponent counter [data :- {:init js/Number} owner]
  (will-mount [_]
              (om/set-state! owner :n (:init data)))
  (render-state [_ {:keys [n]}]
                (html [:div
                       [:span (str "Count: " n)]
                       [:button
                        {:on-click #(om/set-state! owner :n (inc n))}
                        "+"]
                       [:button
                        {:on-click #(om/set-state! owner :n (dec n))}
                        "-"]])))

put es-api and poll-interval in app-state


(ns todogo-cljs.views
  (:require [re-frame.core :as re-frame]
            [reagent.core :refer [dom-node]]
            [todogo-cljs.db :refer [create-todo-list
                                    create-todo
                                    get-todo-lists
                                    get-todos
                                    toggle-todo
                                    delete-todo-list
                                    delete-todo]]
            [todogo-cljs.components :as c]))

(defn todo-lists-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        lists (re-frame/subscribe [:todo-lists])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:span "Home"]]]]
        [:div {:class "col-lg-4 col-md-4"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]]
       c/footer])))

(defn todo-list-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        lists (re-frame/subscribe [:todo-lists])
        todo-list (re-frame/subscribe [:todo-list])
        todos (re-frame/subscribe [:todos])
        todo-text (re-frame/subscribe [:todo-title])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:a {:href "#/"} "Home"]]
          [:li [:span (:title @todo-list)]]]]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]
        [:div {:class "col-lg-4 col-md-4"}
         (c/create-todo-form @todo-list @todo-text)
         (c/todos @todos)]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}]]
       c/footer])))

(defn todo-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        todo (re-frame/subscribe [:todo])
        todo-list (re-frame/subscribe [:todo-list])
        lists (re-frame/subscribe [:todo-lists])
        todos (re-frame/subscribe [:todos])
        todo-text (re-frame/subscribe [:todo-title])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:a {:href "#/"} "Home"]]
          [:li [:a {:href (str "#/lists/" (:id @todo-list))} (:title @todo-list)]]
          [:li [:span (:title @todo)]]]]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-form @todo-list @todo-text)
         (c/todos @todos)]
        [:div {:class "col-lg-4 col-md-4"}
         (c/edit-todo-form @todo)]]
       c/footer])))

(defn about-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])]
    [:div {:class "base-container"}
     (c/nav-bar @main-menu-visible)
     [:div {:class "container"}
      [:h2 "About page"]
      [:div [:a {:href "#/"} "go to the home page"]]]
     c/footer]))

(defn- panels [panel-name]
  ;(println ">>>>>>" panel-name (juxt namespace panel-name))
  (case panel-name
    :todo-lists-panel [todo-lists-panel]
    :todo-list-panel [todo-list-panel]
    :todo-panel [todo-panel]
    :about-panel [about-panel]
    [:div]))

(defn show-panel [panel-name]
  [panels panel-name])

(defn main-panel []
  (let [active-panel (re-frame/subscribe [:active-panel])]
    (fn []
      [show-panel @active-panel])))

(ns todogo-cljs.views
  (:require [re-frame.core :as re-frame]
            [reagent.core :refer [dom-node]]
            [todogo-cljs.db :refer [create-todo-list
                                    create-todo
                                    get-todo-lists
                                    get-todos
                                    toggle-todo
                                    delete-todo-list
                                    delete-todo]]
            [todogo-cljs.components :as c]))

(defn todo-lists-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        lists (re-frame/subscribe [:todo-lists])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:span [:span {:class "fa fa-lg fa-home"}]]]]]
        [:div {:class "col-lg-4 col-md-4"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]]
       c/footer])))

(defn todo-list-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        lists (re-frame/subscribe [:todo-lists])
        todo-list (re-frame/subscribe [:todo-list])
        todos (re-frame/subscribe [:todos])
        todo-text (re-frame/subscribe [:todo-title])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:a {:href "#/"} [:span {:class "fa fa-lg fa-home"}]]]
          [:li [:span (:title @todo-list)]]]]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]
        [:div {:class "col-lg-4 col-md-4"}
         (c/create-todo-form @todo-list @todo-text)
         (c/todos @todos)]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}]]
       c/footer])))

(defn todo-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])
        todo (re-frame/subscribe [:todo])
        todo-list (re-frame/subscribe [:todo-list])
        lists (re-frame/subscribe [:todo-lists])
        todos (re-frame/subscribe [:todos])
        todo-text (re-frame/subscribe [:todo-title])
        todo-list-title (re-frame/subscribe [:todo-list-title])]
    (fn []
      [:div {:class "base-container"}
       (c/nav-bar @main-menu-visible)
       [:div {:class "container"}
        [:div {:class "col-12 "}
         [:ul {:class "breadcrumb"}
          [:li [:a {:href "#/"} [:span {:class "fa fa-lg fa-home"}]]]
          [:li [:a {:href (str "#/lists/" (:id @todo-list))} (:title @todo-list)]]
          [:li [:span (:title @todo)]]]]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-list-form @todo-list-title)
         (c/todo-lists @lists)]
        [:div {:class "col-lg-4 col-md-4 hidden-sm hidden-xs"}
         (c/create-todo-form @todo-list @todo-text)
         (c/todos @todos)]
        [:div {:class "col-lg-4 col-md-4"}
         (c/edit-todo-form @todo)]]
       c/footer])))

(defn about-panel []
  (let [main-menu-visible (re-frame/subscribe [:main-menu-visible])]
    [:div {:class "base-container"}
     (c/nav-bar @main-menu-visible)
     [:div {:class "container"}
      [:h2 "About page"]
      [:div [:a {:href "#/"} "go to the home page"]]]
     c/footer]))

(defn- panels [panel-name]
  ;(println ">>>>>>" panel-name (juxt namespace panel-name))
  (case panel-name
    :todo-lists-panel [todo-lists-panel]
    :todo-list-panel [todo-list-panel]
    :todo-panel [todo-panel]
    :about-panel [about-panel]
    [:div]))

(defn show-panel [panel-name]
  [panels panel-name])

(defn main-panel []
  (let [active-panel (re-frame/subscribe [:active-panel])]
    (fn []
      [show-panel @active-panel])))

Change home breadcrumb to an icon instead of text


(ns in-container.pipeline
  (:use clojure.test)
  (:require [immutant.messaging.pipeline :as pl]
            [immutant.messaging          :as msg]))

(defn pio [f]
  (fn [i]
    (println "INPUT:" i)
    (let [o (f i)]
      (println "OUTPUT:" o)
      o)))

(defn random-queue []
  (msg/as-queue (str (java.util.UUID/randomUUID))))

(defn dollarizer [s]
  (.replace s "S" "$"))

(defn sleeper [x]
  (Thread/sleep 500)
  x)

(deftest it-should-work
  (let [result-queue (random-queue)
        pl (pl/pipeline
            "basic"
            #(.replace % "m" "x")
            (memfn toUpperCase)
            dollarizer
            #(.replace % "$" "Ke$ha")
            #(msg/publish result-queue %))]
    (msg/start result-queue)
    (msg/publish pl "hambiscuit")
    (is (= "HAXBIKe$haCUIT" (msg/receive result-queue)))))

(deftest it-should-work-with-a-result-queue
  (let [result-queue "queue.pl.result-opt"
        pl (pl/pipeline "result-queue"
                        #(.replace % "y" "x")
                        (memfn toUpperCase)
                        dollarizer
                        #(.replace % "$" "NipseyHu$$le")
                        :result-destination result-queue)]
    (msg/publish pl "gravybiscuit")
    (is (= "GRAVXBINipseyHu$$leCUIT" (msg/receive result-queue)))))

(deftest it-should-work-with-concurrency
  (let [result-queue (random-queue)
        pl (pl/pipeline
            "concurrency"
            dollarizer
            (pl/step sleeper :concurrency 10)
            :result-destination result-queue)]
    (dotimes [n 10]
      (msg/publish pl "hamboneS"))
    (let [results
          (keep identity
                (map (fn [_] (msg/receive result-queue :timeout 510))
                     (range 10)))]
      (is (= 10 (count results)))
      (is (= (take 10 (repeat "hambone$")) results)))))

(deftest *pipeline*-should-be-bound
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "pipeline var"
              (fn [_] (msg/publish result-queue (str pl/*pipeline*))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= (str pl) (msg/receive result-queue)))))

(defn chucker [_]
  (throw (Exception. "boom")))

(testing "error handling"
  (deftest global-error-handling-should-work
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "global-eh"
              chucker
              :error-handler (fn [e m]
                               (msg/publish result-queue "caught it!")))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "caught it!" (msg/receive result-queue)))))

  (deftest step-error-handling-should-work
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "step-eh"
              (pl/step chucker
                       :error-handler (fn [e m]
                                        (msg/publish result-queue "from step"))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "from step" (msg/receive result-queue)))))

  (deftest step-error-handling-should-override-global
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "step-eh-override"
              (pl/step chucker
                       :error-handler (fn [e m]
                                        (msg/publish result-queue "from step")))
              :error-handler (fn [e m]
                               (msg/publish result-queue "from global")))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "from step" (msg/receive result-queue)))))

  (deftest *pipeline*-should-be-bound-in-an-error-handler
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "pipeline var in eh"
              chucker
              :error-handler (fn [e m]
                               (msg/publish result-queue (str pl/*pipeline*))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= (str pl) (msg/receive result-queue))))))

(ns in-container.pipeline
  (:use clojure.test)
  (:require [immutant.messaging.pipeline :as pl]
            [immutant.messaging          :as msg]))

(defn pio [f]
  (fn [i]
    (println "INPUT:" i)
    (let [o (f i)]
      (println "OUTPUT:" o)
      o)))

(defn random-queue []
  (msg/as-queue (str (java.util.UUID/randomUUID))))

(defn dollarizer [s]
  (.replace s "S" "$"))

(defn make-sleeper [ms]
  (fn [x]
    (Thread/sleep ms)
    x))

(deftest it-should-work
  (let [result-queue (random-queue)
        pl (pl/pipeline
            "basic"
            #(.replace % "m" "x")
            (memfn toUpperCase)
            dollarizer
            #(.replace % "$" "Ke$ha")
            #(msg/publish result-queue %))]
    (msg/start result-queue)
    (msg/publish pl "hambiscuit")
    (is (= "HAXBIKe$haCUIT" (msg/receive result-queue)))))

(deftest it-should-work-with-a-result-queue
  (let [result-queue "queue.pl.result-opt"
        pl (pl/pipeline "result-queue"
                        #(.replace % "y" "x")
                        (memfn toUpperCase)
                        dollarizer
                        #(.replace % "$" "NipseyHu$$le")
                        :result-destination result-queue)]
    (msg/publish pl "gravybiscuit")
    (is (= "GRAVXBINipseyHu$$leCUIT" (msg/receive result-queue)))))

#_(deftest it-should-work-with-concurrency
  (let [result-queue (random-queue)
        pl (pl/pipeline
            "concurrency"
            dollarizer
            (pl/step (make-sleeper 500) :concurrency 10)
            :result-destination result-queue)]
    (dotimes [n 10]
      (msg/publish pl "hamboneS"))
    (let [results
          (keep identity
                (map (fn [_] (msg/receive result-queue :timeout 510))
                     (range 10)))]
      (is (= 10 (count results)))
      (is (= (take 10 (repeat "hambone$")) results)))))

(deftest *pipeline*-should-be-bound
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "pipeline var"
              (fn [_] (msg/publish result-queue (str pl/*pipeline*))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= (str pl) (msg/receive result-queue)))))

(defn chucker [_]
  (throw (Exception. "boom")))

(testing "error handling"
  (deftest global-error-handling-should-work
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "global-eh"
              chucker
              :error-handler (fn [e m]
                               (msg/publish result-queue "caught it!")))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "caught it!" (msg/receive result-queue)))))

  (deftest step-error-handling-should-work
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "step-eh"
              (pl/step chucker
                       :error-handler (fn [e m]
                                        (msg/publish result-queue "from step"))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "from step" (msg/receive result-queue)))))

  (deftest step-error-handling-should-override-global
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "step-eh-override"
              (pl/step chucker
                       :error-handler (fn [e m]
                                        (msg/publish result-queue "from step")))
              :error-handler (fn [e m]
                               (msg/publish result-queue "from global")))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= "from step" (msg/receive result-queue)))))

  (deftest *pipeline*-should-be-bound-in-an-error-handler
    (let [result-queue (random-queue)
          pl (pl/pipeline
              "pipeline var in eh"
              chucker
              :error-handler (fn [e m]
                               (msg/publish result-queue (str pl/*pipeline*))))]
      (msg/start result-queue)
      (msg/publish pl "hi")
      (is (= (str pl) (msg/receive result-queue))))))

Comment out flaky pipeline test for now.


; Copyright © 2013 - 2016 Dr. Thomas Schank <Thomas.Schank@AlgoCon.ch>
; Licensed under the terms of the GNU Affero General Public License v3.
; See the "LICENSE.txt" file provided with this software.

(ns cider-ci.repository.web
  (:require
    [cider-ci.auth.authorize :as authorize]
    [cider-ci.auth.http-basic :as http-basic]
    [cider-ci.repository.git.repositories :as git.repositories]
    [cider-ci.repository.project-configuration :as project-configuration]
    [cider-ci.repository.repositories :as repositories]
    [cider-ci.repository.sql.repository :as sql.repository]
    [cider-ci.utils.config :as config :refer [get-config]]
    [cider-ci.utils.routing :as routing]
    [cider-ci.utils.status :as status]

    [clj-time.core :as time]
    [clojure.data :as data]
    [clojure.data.json :as json]
    [compojure.core :as cpj]
    [compojure.handler :as cpj.handler]
    [ring.adapter.jetty :as jetty]
    [ring.middleware.json]
    [ring.middleware.params]
    [ring.util.response :refer [charset]]
    [ring.util.response]

    [clj-logging-config.log4j :as logging-config]
    [clojure.tools.logging :as logging]
    [logbug.debug :as debug :refer [I> I>>]]
    [logbug.ring :refer [wrap-handler-with-logging]]
    [logbug.thrown :as thrown]
    ))

;##### get file ###############################################################

(defn get-git-file [request]
  (logging/debug get-git-file [request])
  (let [repository-id (:id (:route-params request))
        relative-web-path (:* (:route-params request))
        relative-file-path (str (-> (get-config) :services :repository :repositories :path) "/" repository-id "/" relative-web-path)
        file (clojure.java.io/file relative-file-path)
        abs-path (.getAbsolutePath file)]
    (logging/debug {:repositories-id repository-id
                    :relative-file-path relative-file-path
                    :abs-path abs-path
                    :file-exists? (.exists file)})
    (if (.exists file)
      (ring.util.response/file-response relative-file-path nil)
      {:status 404})))

(defn respond-with-500 [request ex]
  (logging/warn "RESPONDING WITH 500" {:exception (thrown/stringify ex) :request request})
  {:status 500 :body (thrown/stringify ex)})

(defn get-path-content [request]
  (logging/debug request)
  (try
    (let [id (-> request :route-params :id)
          path (-> request :route-params :*)]
      (when-let [repository (sql.repository/resolve id)]
        (when-let [content  (git.repositories/get-path-contents repository id path)]
          {:body content})))
    (catch clojure.lang.ExceptionInfo e
      (cond (re-find #"does not exist in"  (str e)) {:status 404 :body (-> e ex-data :err)}
            :else (respond-with-500 request e)))
    (catch Exception e
      (respond-with-500 request e))))

;#### repository update notification ##########################################

(defn update-notification-handler [request]
  (if-let [repository (sql.repository/get-repository-by-update-notification-token
                        (-> request :params :update_notification_token))]
    (do (repositories/update-repository repository)
      {:status 202 :body "OK"})
    {:status 404 :body "The corresponding repository was not found"}))

(defn wrap-repositories-update-notifications [default-handler]
  (cpj/routes
    (cpj/POST "/update-notification/:update_notification_token"
              _ #'update-notification-handler)
    (cpj/ANY "*" _ default-handler)))

;##### project configuration ##################################################

(defn get-project-configuration [request]
  (logging/info request)
  (-> (try
        (when-let [content (project-configuration/build-project-configuration
                             (-> request :params :id))]
          {:body (json/write-str content :key-fn #(subs (str %) 1))
           :headers {"Content-Type" "application/json"}})
        (catch clojure.lang.ExceptionInfo e
          (case (-> e ex-data :status )
            404 {:status 404
                 :headers {"Content-Type" "application/json"}
                 :body (json/write-str (ex-data e)) }
            422 {:status 422
                 :body (thrown/stringify e)}
            (respond-with-500 request e)))
        (catch Throwable e
          (respond-with-500 request e)))
      (charset "UTF-8")))

;##### routes #################################################################

(def routes
  (cpj/routes
    (cpj/GET "/project-configuration/:id" _ get-project-configuration)
    (cpj/GET "/path-content/:id/*" _ get-path-content)
    (cpj/GET "/:id/git/*" _ get-git-file )))

(defn build-main-handler [context]
  (I> wrap-handler-with-logging
      (cpj.handler/api routes)
      routing/wrap-shutdown
      (ring.middleware.params/wrap-params)
      (ring.middleware.json/wrap-json-params)
      status/wrap
      (authorize/wrap-require! {:service true})
      (http-basic/wrap {:service true})
      wrap-repositories-update-notifications
      (routing/wrap-prefix context)
      (routing/wrap-log-exception)))

;#### debug ###################################################################
;(logging-config/set-logger! :level :debug)
;(logging-config/set-logger! :level :info)
;(debug/debug-ns 'cider-ci.auth.http-basic)
;(debug/debug-ns *ns*)

; Copyright © 2013 - 2016 Dr. Thomas Schank <Thomas.Schank@AlgoCon.ch>
; Licensed under the terms of the GNU Affero General Public License v3.
; See the "LICENSE.txt" file provided with this software.

(ns cider-ci.repository.web
  (:require
    [cider-ci.auth.authorize :as authorize]
    [cider-ci.auth.http-basic :as http-basic]
    [cider-ci.repository.git.repositories :as git.repositories]
    [cider-ci.repository.project-configuration :as project-configuration]
    [cider-ci.repository.repositories :as repositories]
    [cider-ci.repository.sql.repository :as sql.repository]
    [cider-ci.utils.config :as config :refer [get-config]]
    [cider-ci.utils.routing :as routing]
    [cider-ci.utils.status :as status]

    [clj-time.core :as time]
    [clojure.data :as data]
    [clojure.data.json :as json]
    [compojure.core :as cpj]
    [compojure.handler :as cpj.handler]
    [ring.adapter.jetty :as jetty]
    [ring.middleware.json]
    [ring.middleware.params]
    [ring.util.response :refer [charset]]
    [ring.util.response]

    [clj-logging-config.log4j :as logging-config]
    [clojure.tools.logging :as logging]
    [logbug.debug :as debug :refer [I> I>>]]
    [logbug.ring :refer [wrap-handler-with-logging]]
    [logbug.thrown :as thrown]
    ))

;##### get file ###############################################################

(defn get-git-file [request]
  (logging/debug get-git-file [request])
  (let [repository-id (:id (:route-params request))
        relative-web-path (:* (:route-params request))
        relative-file-path (str (-> (get-config) :services :repository :repositories :path) "/" repository-id "/" relative-web-path)
        file (clojure.java.io/file relative-file-path)
        abs-path (.getAbsolutePath file)]
    (logging/debug {:repositories-id repository-id
                    :relative-file-path relative-file-path
                    :abs-path abs-path
                    :file-exists? (.exists file)})
    (if (.exists file)
      (ring.util.response/file-response relative-file-path nil)
      {:status 404})))

(defn respond-with-500 [request ex]
  (logging/warn "RESPONDING WITH 500" {:exception (thrown/stringify ex) :request request})
  {:status 500 :body (thrown/stringify ex)})

(defn get-path-content [request]
  (logging/debug request)
  (try
    (let [id (-> request :route-params :id)
          path (-> request :route-params :*)]
      (when-let [repository (sql.repository/resolve id)]
        (when-let [content  (git.repositories/get-path-contents repository id path)]
          {:body content})))
    (catch clojure.lang.ExceptionInfo e
      (cond (re-find #"does not exist in"  (str e)) {:status 404 :body (-> e ex-data :err)}
            :else (respond-with-500 request e)))
    (catch Exception e
      (respond-with-500 request e))))

;#### repository update notification ##########################################

(defn update-notification-handler [request]
  (if-let [repository (sql.repository/get-repository-by-update-notification-token
                        (-> request :params :update_notification_token))]
    (do (repositories/update-repository repository)
      {:status 202 :body "OK"})
    {:status 404 :body "The corresponding repository was not found"}))

(defn wrap-repositories-update-notifications [default-handler]
  (cpj/routes
    (cpj/POST "/update-notification/:update_notification_token"
              _ #'update-notification-handler)
    (cpj/ANY "*" _ default-handler)))

;##### project configuration ##################################################

(defn get-project-configuration [request]
  (-> (try
        (when-let [content (project-configuration/build-project-configuration
                             (-> request :params :id))]
          {:body (json/write-str content :key-fn #(subs (str %) 1))
           :headers {"Content-Type" "application/json"}})
        (catch clojure.lang.ExceptionInfo e
          (case (-> e ex-data :status )
            404 {:status 404
                 :headers {"Content-Type" "application/json"}
                 :body (json/write-str (ex-data e)) }
            422 {:status 422
                 :body (thrown/stringify e)}
            (respond-with-500 request e)))
        (catch Throwable e
          (respond-with-500 request e)))
      (charset "UTF-8")))

;##### routes #################################################################

(def routes
  (cpj/routes
    (cpj/GET "/project-configuration/:id" _ get-project-configuration)
    (cpj/GET "/path-content/:id/*" _ get-path-content)
    (cpj/GET "/:id/git/*" _ get-git-file )))

(defn build-main-handler [context]
  (I> wrap-handler-with-logging
      (cpj.handler/api routes)
      routing/wrap-shutdown
      (ring.middleware.params/wrap-params)
      (ring.middleware.json/wrap-json-params)
      status/wrap
      (authorize/wrap-require! {:service true})
      (http-basic/wrap {:service true})
      wrap-repositories-update-notifications
      (routing/wrap-prefix context)
      (routing/wrap-log-exception)))

;#### debug ###################################################################
;(logging-config/set-logger! :level :debug)
;(logging-config/set-logger! :level :info)
;(debug/debug-ns 'cider-ci.auth.http-basic)
;(debug/debug-ns *ns*)

Remove logging statement

Remove logging statement


(ns oarlock.validation
  (:require [schema.core :as s]))

(def validations
  {:task {:id-sk        s/Str
          :id-sk-origin s/Keyword
          :name         s/Str
          :version      s/Str
          :description  s/Str}
   :perf-asmt {:id-sk                    s/Str
               :id-sk-origin             s/Keyword
               (s/optional-key :name)    s/Str
               :version                  s/Str
               :type                     s/Keyword
               :duration-rating-days     s/Int
               :comps                    [s/Str]
               :credit-value-numerator   s/Int
               :credit-value-denominator s/Int}
   :student2perf-asmt {:user  s/Str
                       :task  s/Str
                       :grade s/Bool}})

(defn validator
  [entity-type data]
  (let [validation (entity-type validations)]
    (try
      (s/validate
       validation
       data)
      (catch Exception e (.getMessage e)))))

(ns oarlock.validation
  (:require [schema.core :as s]))

(def validations
  {:task {:id-sk        s/Str
          :id-sk-origin s/Keyword
          :name         s/Str
          :version      s/Str
          :description  s/Str}
   :perf-asmt {:id-sk                    s/Str
               :id-sk-origin             s/Keyword
               (s/optional-key :name)    s/Str
               :version                  s/Str
               :type                     s/Keyword
               :duration-rating-days     s/Int
               :comps                    [s/Str]
               :credit-value-numerator   s/Int
               :credit-value-denominator s/Int}
   :student2perf-asmt {:id-sk           s/Str
                       :id-sk-origin    s/Str
                       :user            s/Str
                       :task            s/Str
                       :grade           s/Bool}})

(defn validator
  [entity-type data]
  (let [validation (entity-type validations)]
    (try
      (s/validate
       validation
       data)
      (catch Exception e (.getMessage e)))))

add id-sk and id-sk-origin to student2perf-asmt validation


(ns cljam.t-core
  (:use midje.sweet
        cljam.t-common)
  (:require [clojure.java.io :as io]
            [cljam.core :as core]))

(defmacro with-out-file
  [f & body]
  `(binding [*out* (clojure.java.io/writer ~f)]
     ~@body))

(def temp-out (str temp-dir "/out"))
(def temp-bam (str temp-dir "/out.bam"))
(def temp-sam (str temp-dir "/out.sam"))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about view"
        (with-out-file temp-out (core/view [test-sam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.view")
        (with-out-file temp-out (core/view [test-bam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.view")
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about convert"
        ;; sam => bam
        (core/convert [test-sam-file temp-bam]) => anything
        (slurp-bam-for-test temp-bam) => (slurp-sam-for-test test-sam-file)
        (slurp-bam-for-test temp-bam) => (slurp-bam-for-test test-bam-file)
        ;; bam => sam
        (core/convert [test-bam-file temp-sam]) => anything
        (slurp-sam-for-test temp-sam) => (slurp-bam-for-test test-bam-file)
        (slurp-sam-for-test temp-sam) => (slurp-sam-for-test test-sam-file)
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about sort (by pos)"
        ;; see https://gitlab.xcoo.jp/chrovis/cljam/issues/12
        (with-out-file temp-out (core/sort ["-o" "coordinate" test-sam-file temp-sam])) =future=> anything
        (slurp-sam-for-test temp-sam) =future=> test-sam-sorted-by-pos
        (check-sort-order (slurp-sam-for-test temp-sam) test-sam-sorted-by-pos) =future=> anything
        (with-out-file temp-out (core/sort ["-o" "coordinate" test-bam-file temp-bam])) => anything
        (slurp-bam-for-test temp-bam) => test-sam-sorted-by-pos
        (check-sort-order (slurp-bam-for-test temp-bam) test-sam-sorted-by-pos) => anything
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about sort (by qname)"
        (with-out-file temp-out (core/sort ["-o" "queryname" test-sam-file temp-sam])) =future=> anything
        (slurp-sam-for-test temp-sam) =future=> test-sam-sorted-by-qname
        (with-out-file temp-out (core/sort ["-o" "queryname" test-bam-file temp-bam])) =future=> anything
        (slurp-bam-for-test temp-bam) =future=> test-sam-sorted-by-qname
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about index"
        "TODO" =future=> nil))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about pileup"
        (with-out-file temp-out (core/pileup [test-sorted-bam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.pileup")
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about faidx"
        "TODO" =future=> nil))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about dict"
        "TODO" =future=> nil))

(ns cljam.t-core
  (:use midje.sweet
        cljam.t-common)
  (:require [clojure.java.io :as io]
            [cljam.core :as core]))

(defmacro with-out-file
  [f & body]
  `(binding [*out* (clojure.java.io/writer ~f)]
     ~@body))

(def temp-out (str temp-dir "/out"))
(def temp-bam (str temp-dir "/out.bam"))
(def temp-sam (str temp-dir "/out.sam"))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about view"
        (with-out-file temp-out (core/view [test-sam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.view")
        (with-out-file temp-out (core/view [test-bam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.view")
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about convert"
        ;; sam => bam
        (core/convert [test-sam-file temp-bam]) => anything
        (slurp-bam-for-test temp-bam) => (slurp-sam-for-test test-sam-file)
        (slurp-bam-for-test temp-bam) => (slurp-bam-for-test test-bam-file)
        ;; bam => sam
        (core/convert [test-bam-file temp-sam]) => anything
        (slurp-sam-for-test temp-sam) => (slurp-bam-for-test test-bam-file)
        (slurp-sam-for-test temp-sam) => (slurp-sam-for-test test-sam-file)
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about sort (by pos)"
        ;; see https://gitlab.xcoo.jp/chrovis/cljam/issues/12
        (with-out-file temp-out (core/sort ["-o" "coordinate" test-sam-file temp-sam])) =future=> anything
        (slurp-sam-for-test temp-sam) =future=> test-sam-sorted-by-pos
        (check-sort-order (slurp-sam-for-test temp-sam) test-sam-sorted-by-pos) =future=> anything
        (with-out-file temp-out (core/sort ["-o" "coordinate" test-bam-file temp-bam])) => anything
        (slurp-bam-for-test temp-bam) => test-sam-sorted-by-pos
        (check-sort-order (slurp-bam-for-test temp-bam) test-sam-sorted-by-pos) => anything
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about sort (by qname)"
        (with-out-file temp-out (core/sort ["-o" "queryname" test-sam-file temp-sam])) =future=> anything
        (slurp-sam-for-test temp-sam) =future=> test-sam-sorted-by-qname
        (with-out-file temp-out (core/sort ["-o" "queryname" test-bam-file temp-bam])) =future=> anything
        (slurp-bam-for-test temp-bam) =future=> test-sam-sorted-by-qname
        ))

(with-state-changes [(before :facts (do (prepare-cache!)
                                        (io/copy (io/file test-sorted-bam-file)
                                                 (io/file temp-bam))))
                     (after  :facts (clean-cache!))]
  (fact "about index"
        (with-out-file temp-out (core/index [temp-bam])) => anything
        (.exists (io/file (str temp-bam ".bai"))) => truthy
        ))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (fact "about pileup"
        (with-out-file temp-out (core/pileup [test-sorted-bam-file])) => anything
        (slurp temp-out) => (slurp "test/resources/t_core.pileup")
        ))

(with-state-changes [(before :facts (do (prepare-cache!)
                                        (io/copy (io/file test-fa-file)
                                                 (io/file temp-out))))
                     (after  :facts (clean-cache!))]
  (fact "about faidx"
        (with-out-file temp-out (core/faidx [temp-out])) => anything
        (.exists (io/file (str temp-out ".fai"))) => truthy))

(with-state-changes [(before :facts (prepare-cache!))
                     (after  :facts (clean-cache!))]
  (let [temp-dict (str temp-dir "/out.dict")]
    (fact "about dict"
          (with-out-file temp-out (core/dict [test-fa-file temp-dict])) =future=> anything
          (.exists (io/file temp-dict)) =future=> truthy)))

add more tests

add more tests


(ns uio.test-uio
  (:require [uio.uio :refer :all]
            [uio.impl :refer [url->ext+s->s intercalate-with-dirs ensure-has-no-trailing-slash]]
            [midje.sweet :refer :all])
  (:import (java.util.zip GZIPOutputStream)
           (org.apache.commons.compress.compressors CompressorStreamFactory)))

(facts "URL manipulation fns are working"
  (scheme        "foo://user@host:8080/some-dir/file.txt?arg=value") => :foo
  (host          "foo://user@host:8080/some-dir/file.txt?arg=value") => "host"
  (port          "foo://user@host:8080/some-dir/file.txt?arg=value") => 8080
  (path          "foo://user@host:8080/some-dir/file.txt?arg=value") => "/some-dir/file.txt"
  (path-no-slash "foo://user@host:8080/some-dir/file.txt?arg=value") => "some-dir/file.txt"

  (ensure-has-no-trailing-slash "test")                              => "test"
  (ensure-has-no-trailing-slash "test/")                             => "test"
  (ensure-has-no-trailing-slash "test///")                           => "test")

(facts "In-memory implementation works"
  (spit  (to   "mem:///greeetings.txt") "hello") => nil
  (slurp (from "mem:///greeetings.txt"))         => "hello")

(facts "Deducing of (de)compression codecs works, even for chained ones"
  (map first (url->ext+s->s ext->is->is "hdfs:///far-away/and/well-archived.xz.bz2.gz")) => [:gz :bz2 :xz]
  (map first (url->ext+s->s ext->os->os "sftp:///far-away/and/well-archived.xz.bz2.gz")) => [:gz :bz2 :xz]
  (map first (url->ext+s->s ext->os->os "mem:///dont-forget-leading-slash.txt.gz"))   => [:gz]

  ; unknown codecs between known ones
  (url->ext+s->s ext->is->is "hdfs:///far-away/and/well-archived.xz.ufoz.bz2.gz") => (throws Exception #"Got at least one unsupported codec")
  (url->ext+s->s ext->os->os "sftp:///far-away/and/well-archived.xz.ufoz.bz2.gz") => (throws Exception #"Got at least one unsupported codec"))

(facts "(De)compression works, even for chained extensions"
  (dorun
    (map (fn [[url content os->compressing-os]]
           (spit (to* url) content)                         ; put

           (seq (is->bytes (from url)))                     ; ensure that stored content was actually compressed
           => (seq (with-baos->bytes #(with-open [os (os->compressing-os %)]
                                        (.write os (.getBytes content)))))

           (slurp (from* url)) => content)                  ; get

         ; [ [url content os->compressing-os]
         [["mem:///file.txt"            "I am plain text"   identity]
          ["mem:///file.txt.gz"         "I am gzipped text" #(GZIPOutputStream. %)]
          ["mem:///file.txt.bz2"        "I am bzipped text" #(.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2 %)]
          ["mem:///file.txt.xz"         "I am xzipped text" #(.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/XZ %)]
          ["mem:///file1.txt.xz.bz2.gz" "I am xzipped, bzipped and gzipped text"
           #(->> (GZIPOutputStream. %)
                 (.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2)
                 (.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/XZ))]])))

(facts "intercalate-with-dirs works"
  ; base case
  (intercalate-with-dirs [])                         => []

  ; base case + 1
  (intercalate-with-dirs [{:url "1.txt"}])           => [{:url "1.txt"}]

  ; base case + 2 + flush
  (intercalate-with-dirs [{:url "1.txt"}
                          {:url "123/2.txt"}])       => [{:url "1.txt"}
                                                             {:url "123" :dir true}
                                                             {:url "123/2.txt"}]
  ; simple case + continue + skip matching last flushed dir
  (intercalate-with-dirs "123" [{:url "1.txt"}
                                {:url "123/2.txt"}]) => [{:url "1.txt"}
                                                         {:url "123/2.txt"}]
  ; simple case + continue
  (intercalate-with-dirs "123" [{:url "456/1.txt"}
                                {:url "456/2.txt"}]) => [{:url "456" :dir true}
                                                         {:url "456/1.txt"}
                                                         {:url "456/2.txt"}]
  ; complex case
  (intercalate-with-dirs [{:url "1.txt"}
                          {:url "123.txt"}
                          ; 123
                          {:url "123/1.txt"}
                          {:url "123/2.txt"}
                          {:url "123/3.txt"}
                          ; 123/123
                          {:url "123/123/1.txt"}
                          ; 456
                          {:url "456/1.txt"}
                          ; 456/123
                          {:url "456/123/1.txt"}
                          {:url "456/123/2.txt"}
                          ; 456/456
                          {:url "456/456/3.txt"}
                          {:url "456/5.txt"}
                          {:url "789.txt"}])         => [{:url "1.txt"}
                                                         {:url "123.txt"}
                                                         {:url "123" :dir true}
                                                         {:url "123/1.txt"}
                                                         {:url "123/2.txt"}
                                                         {:url "123/3.txt"}
                                                         {:url "123/123" :dir true}
                                                         {:url "123/123/1.txt"}
                                                         {:url "456" :dir true}
                                                         {:url "456/1.txt"}
                                                         {:url "456/123" :dir true}
                                                         {:url "456/123/1.txt"}
                                                         {:url "456/123/2.txt"}
                                                         {:url "456/456" :dir true}
                                                         {:url "456/456/3.txt"}
                                                         {:url "456/5.txt"}
                                                         {:url "789.txt"}])

(ns uio.test-uio
  (:require [uio.uio :refer :all]
            [uio.impl :refer [ensure-has-no-trailing-slash
                              intercalate-with-dirs
                              url->ext+s->s]]
            [midje.sweet :refer :all])
  (:import (java.util.zip GZIPOutputStream)
           (org.apache.commons.compress.compressors CompressorStreamFactory)))

(facts "URL manipulation fns are working"
  (scheme        "foo://user@host:8080/some-dir/file.txt?arg=value") => :foo
  (host          "foo://user@host:8080/some-dir/file.txt?arg=value") => "host"
  (port          "foo://user@host:8080/some-dir/file.txt?arg=value") => 8080
  (path          "foo://user@host:8080/some-dir/file.txt?arg=value") => "/some-dir/file.txt"
  (path-no-slash "foo://user@host:8080/some-dir/file.txt?arg=value") => "some-dir/file.txt"

  (normalize     "file:///")                                         => "file:///"
  (normalize     "file:////")                                        => "file:///"
  (normalize     "file://///")                                       => "file:///"
  (normalize     "file://host/path/to")                              => "file://host/path/to"
  (normalize     "file://host/path/to/")                             => "file://host/path/to/"
  (normalize     "file://host/path/to//")                            => "file://host/path/to/"

  (ensure-has-no-trailing-slash "test")                              => "test"
  (ensure-has-no-trailing-slash "test/")                             => "test"
  (ensure-has-no-trailing-slash "test///")                           => "test")

(facts "In-memory implementation works"
  (spit  (to   "mem:///greeetings.txt") "hello") => nil
  (slurp (from "mem:///greeetings.txt"))         => "hello")

(facts "Deducing of (de)compression codecs works, even for chained ones"
  (map first (url->ext+s->s ext->is->is "hdfs:///far-away/and/well-archived.xz.bz2.gz")) => [:gz :bz2 :xz]
  (map first (url->ext+s->s ext->os->os "sftp:///far-away/and/well-archived.xz.bz2.gz")) => [:gz :bz2 :xz]
  (map first (url->ext+s->s ext->os->os "mem:///dont-forget-leading-slash.txt.gz"))   => [:gz]

  ; unknown codecs between known ones
  (url->ext+s->s ext->is->is "hdfs:///far-away/and/well-archived.xz.ufoz.bz2.gz") => (throws Exception #"Got at least one unsupported codec")
  (url->ext+s->s ext->os->os "sftp:///far-away/and/well-archived.xz.ufoz.bz2.gz") => (throws Exception #"Got at least one unsupported codec"))

(facts "(De)compression works, even for chained extensions"
  (dorun
    (map (fn [[url content os->compressing-os]]
           (spit (to* url) content)                         ; put

           (seq (is->bytes (from url)))                     ; ensure that stored content was actually compressed
           => (seq (with-baos->bytes #(with-open [os (os->compressing-os %)]
                                        (.write os (.getBytes content)))))

           (slurp (from* url)) => content)                  ; get

         ; [ [url content os->compressing-os]
         [["mem:///file.txt"            "I am plain text"   identity]
          ["mem:///file.txt.gz"         "I am gzipped text" #(GZIPOutputStream. %)]
          ["mem:///file.txt.bz2"        "I am bzipped text" #(.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2 %)]
          ["mem:///file.txt.xz"         "I am xzipped text" #(.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/XZ %)]
          ["mem:///file1.txt.xz.bz2.gz" "I am xzipped, bzipped and gzipped text"
           #(->> (GZIPOutputStream. %)
                 (.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/BZIP2)
                 (.createCompressorOutputStream (CompressorStreamFactory.) CompressorStreamFactory/XZ))]])))

(facts "intercalate-with-dirs works"
  ; base case
  (intercalate-with-dirs [])                         => []

  ; base case + 1
  (intercalate-with-dirs [{:url "1.txt"}])           => [{:url "1.txt"}]

  ; base case + 2 + flush
  (intercalate-with-dirs [{:url "1.txt"}
                          {:url "123/2.txt"}])       => [{:url "1.txt"}
                                                             {:url "123" :dir true}
                                                             {:url "123/2.txt"}]
  ; simple case + continue + skip matching last flushed dir
  (intercalate-with-dirs "123" [{:url "1.txt"}
                                {:url "123/2.txt"}]) => [{:url "1.txt"}
                                                         {:url "123/2.txt"}]
  ; simple case + continue
  (intercalate-with-dirs "123" [{:url "456/1.txt"}
                                {:url "456/2.txt"}]) => [{:url "456" :dir true}
                                                         {:url "456/1.txt"}
                                                         {:url "456/2.txt"}]
  ; complex case
  (intercalate-with-dirs [{:url "1.txt"}
                          {:url "123.txt"}
                          ; 123
                          {:url "123/1.txt"}
                          {:url "123/2.txt"}
                          {:url "123/3.txt"}
                          ; 123/123
                          {:url "123/123/1.txt"}
                          ; 456
                          {:url "456/1.txt"}
                          ; 456/123
                          {:url "456/123/1.txt"}
                          {:url "456/123/2.txt"}
                          ; 456/456
                          {:url "456/456/3.txt"}
                          {:url "456/5.txt"}
                          {:url "789.txt"}])         => [{:url "1.txt"}
                                                         {:url "123.txt"}
                                                         {:url "123" :dir true}
                                                         {:url "123/1.txt"}
                                                         {:url "123/2.txt"}
                                                         {:url "123/3.txt"}
                                                         {:url "123/123" :dir true}
                                                         {:url "123/123/1.txt"}
                                                         {:url "456" :dir true}
                                                         {:url "456/1.txt"}
                                                         {:url "456/123" :dir true}
                                                         {:url "456/123/1.txt"}
                                                         {:url "456/123/2.txt"}
                                                         {:url "456/456" :dir true}
                                                         {:url "456/456/3.txt"}
                                                         {:url "456/5.txt"}
                                                         {:url "789.txt"}])

(facts "Normalize works")

Fix file:////, file://///

Fix file:////, file://///


{:user {:dependencies [[clj-stacktrace "0.2.5"]
                       [org.clojure/tools.trace "0.7.5"]
                       [redl "0.1.0"]
                       [spyscope "0.1.3"]
                       [slamhound "1.3.3"]]
        :plugins [[lein-difftest "1.3.7"]
                  [lein-drip "0.1.1-SNAPSHOT"]
                  [lein-clojars "0.9.1"]
                  [lein-pprint "1.1.1"]
                  [lein-ring "0.8.0"]
                  [lein-cljsbuild "0.1.9"]
                  [lein-deps-tree "0.1.2"]
                  [lein-marginalia "0.7.1"]]
        :repl-options {:timeout 120000}
        :injections [(require '[redl core complete])
                     (require 'spyscope.core)
                     (let [orig (ns-resolve (doto 'clojure.stacktrace require)
                                            'print-cause-trace)
                           new (ns-resolve (doto 'clj-stacktrace.repl require)
                                           'pst)]
                       (alter-var-root orig (constantly @new)))]
        :vimclojure-opts {:repl true}}}

{:user {:dependencies [[clj-stacktrace "0.2.5"]
                       [org.clojure/tools.trace "0.7.5"]
                       [org.clojure/tools.namespace "0.2.3"]
                       [redl "0.1.0"]
                       [spyscope "0.1.3"]
                       [slamhound "1.3.3"]]
        :plugins [[lein-difftest "1.3.7"]
                  [lein-drip "0.1.1-SNAPSHOT"]
                  [lein-clojars "0.9.1"]
                  [lein-pprint "1.1.1"]
                  [lein-ring "0.8.0"]
                  [lein-cljsbuild "0.1.9"]
                  [lein-deps-tree "0.1.2"]
                  [lein-marginalia "0.7.1"]]
        :repl-options {:timeout 120000}
        :injections [(require '[redl core complete])
                     (require 'spyscope.core)
                     (require 'clojure.tools.namespace)
                     (let [orig (ns-resolve (doto 'clojure.stacktrace require)
                                            'print-cause-trace)
                           new (ns-resolve (doto 'clj-stacktrace.repl require)
                                           'pst)]
                       (alter-var-root orig (constantly @new)))]
        :vimclojure-opts {:repl true}}}

Add clojure.tools.namespace to user deps.


(defproject replete "0.1.0"
  :dependencies [[andare "0.7.0"]                           ; Update in script/build also
                 [cljsjs/parinfer "1.8.1-0"]
                 [com.cognitect/transit-clj "0.8.275"]
                 [com.cognitect/transit-cljs "0.8.220"]
                 [fipp "0.6.8"]
                 [tailrecursion/cljson "1.0.7"]
                 [malabarba/lazy-map "1.1"]
                 [org.clojure/clojure "1.8.0"]
                 [org.clojure/clojurescript "1.9.542"]
                 [org.clojure/test.check "0.9.1-SNAPSHOT"]] ; Clone and build test.check master (ref'd in script/build)
  :clean-targets ["out" "target"]
  :plugins [[lein-cljsbuild "1.1.5"]]
  :cljsbuild {:builds {:test {:source-paths ["src" "test"]
                              :compiler {:output-to "test/resources/compiled.js"
                                         :optimizations :whitespace
                                         :pretty-print true}}}
              :test-commands {"test" ["phantomjs"
                                      "test/resources/test.js"
                                      "test/resources/test.html"]}})

(defproject replete "0.1.0"
  :dependencies [[andare "0.7.0"]                           ; Update in script/build also
                 [cljsjs/parinfer "1.8.1-0"]
                 [com.cognitect/transit-clj "0.8.275"]
                 [com.cognitect/transit-cljs "0.8.220"]
                 [fipp "0.6.8"]
                 [tailrecursion/cljson "1.0.7"]
                 [malabarba/lazy-map "1.1"]
                 [org.clojure/clojure "1.8.0"]
                 [org.clojure/clojurescript "1.9.562"]
                 [org.clojure/test.check "0.9.1-SNAPSHOT"]] ; Clone and build test.check master (ref'd in script/build)
  :clean-targets ["out" "target"]
  :plugins [[lein-cljsbuild "1.1.5"]]
  :cljsbuild {:builds {:test {:source-paths ["src" "test"]
                              :compiler {:output-to "test/resources/compiled.js"
                                         :optimizations :whitespace
                                         :pretty-print true}}}
              :test-commands {"test" ["phantomjs"
                                      "test/resources/test.js"
                                      "test/resources/test.html"]}})

Update to ClojureScript 1.9.562

Update to ClojureScript 1.9.562


(ns podload.downloader
  (:require [feedparser-clj.core :as feedparser]
            [me.raynes.moments :as m]
            [flatland.chronicle :as c]
            [clj-time.local :as l]
            [utilza.misc :as umisc]
            [taoensso.timbre :as log]
            [clj-http.client :as client]
            [clj-time.core :as t]
            [utilza.repl :as urepl])
  (:import  [com.mpatric.mp3agic ID3v2 ID3v24Tag Mp3File]))

(def executor (m/executor 10))

(defn get-latest-url
  "Fetches a podcast supplied as an url, and Returns url of latest enclosure in it."
  [podcast-url]
  (->>  podcast-url
        feedparser/parse-feed
        :entries
        (mapcat :enclosures)
        (map :url)
        first))

(defn write-file!
  "Takes input stream, and output filename, and does the deed."
  [in out]
  (with-open [w (clojure.java.io/output-stream out)]
    (.write w in)))

(defn download-latest!
  "Takes podcast url, grabs latest enclosure, and dumps it to outfile path"
  [podcast-url outfile]
  (-> podcast-url
      get-latest-url
      (client/get  {:as :byte-array})
      :body
      (write-file! outfile)))

(defn tag-and-copy!
  "Takes an infile, an outfile, and an album name,
   brutally removes any tags (because mp3agic was puking on NotSupportedExceptions),
   copies infile to outfile, and adds the album tag to it"
  [infile outfile album-name]
  (let [f (Mp3File.  infile)]
    (when (.hasId3v1Tag f)
      (.removeId3v1Tag f))
    (when (.hasId3v2Tag f)
      (.removeId3v2Tag f))
    (when (.hasCustomTag f)
      (.removeCustomTag f))
    (let [t (ID3v24Tag.)]
      (.setId3v2Tag f t)
      (.setAlbum t album-name)
      (.save f outfile))))

(defn do-everything!
  "Takes a config, downloads the rss feed, finds the most recent file,
   downloads it, tags it, and moves it where it needs to go."
  [{:keys [name feed-url filename tag dest-dir] :as config}]
  (log/info "Beginning " name)
  (let [tempfile  (str "/tmp/" (java.util.UUID/randomUUID))
        destfile (str dest-dir "/" filename)]
    ;; clean up old one first
    (clojure.java.io/delete-file destfile true)
    (try
      (download-latest! feed-url tempfile)
      (tag-and-copy! tempfile destfile tag)
      (log/info "Completed downloading " name)
      (catch  Exception e
        (log/error e name))
      (finally
        (clojure.java.io/delete-file tempfile true)))))

(defn schedule-local
  "Hack until moments supports local time.
   Schedule a task to run based on a Chronicle specification."
  [executor spec f]
  (let [[start & rest] (c/times-for spec (l/local-now))]
    (m/schedule-at executor start (#'me.raynes.moments/chronicle-scheduler executor f rest))))

(defn format-schedule
  [frequency]
  (->> (l/local-now)
       (c/times-for frequency)
       (map #(l/format-local-time % :mysql))
       (take 5)
       (umisc/inter-str "\n")))

(defn schedule-one!
  [{:keys [name frequency] :as config}]
  (log/info "Scheduling " name " at:\n"  (format-schedule frequency) " etc etc...")
  (schedule-local executor frequency #(do-everything! config)))

(defn schedule-all!
  [configs]
  (doseq [c configs]
    (schedule-one! c)))

(defn seed!
  "Start off with forcing all the configs to run."
  [configs]
  (doseq [c configs]
    (do-everything! c)))

(ns podload.downloader
  (:require [feedparser-clj.core :as feedparser]
            [me.raynes.moments :as m]
            [flatland.chronicle :as c]
            [clj-time.local :as l]
            [utilza.misc :as umisc]
            [taoensso.timbre :as log]
            [clj-http.client :as client]
            [clj-time.core :as t]
            [utilza.repl :as urepl])
  (:import  [com.mpatric.mp3agic ID3v2 ID3v24Tag Mp3File]))

(def executor (m/executor 10))

(defn get-latest-url
  "Fetches a podcast supplied as an url, and Returns url of latest enclosure in it."
  [podcast-url]
  (->>  podcast-url
        feedparser/parse-feed
        :entries
        (mapcat :enclosures)
        (map :url)
        first))

(defn write-file!
  "Takes input stream, and output filename, and does the deed."
  [in out]
  (with-open [w (clojure.java.io/output-stream out)]
    (.write w in)))

(defn download-latest!
  "Takes podcast url, grabs latest enclosure, and dumps it to outfile path"
  [podcast-url outfile]
  (-> podcast-url
      get-latest-url
      (client/get  {:as :byte-array})
      :body
      (write-file! outfile)))

(defn tag-and-copy!
  "Takes an infile, an outfile, and an album name,
   brutally removes any tags (because mp3agic was puking on NotSupportedExceptions),
   copies infile to outfile, and adds the album tag to it"
  [infile outfile album-name]
  (let [f (Mp3File.  infile)
        t (ID3v24Tag.)]
    (when (.hasId3v1Tag f)
      (.removeId3v1Tag f))
    (when (.hasId3v2Tag f)
      (.removeId3v2Tag f))
    (when (.hasCustomTag f)
      (.removeCustomTag f))
    (.setId3v2Tag f t)
    (.setAlbum t album-name)
    (.save f outfile)))

(defn do-everything!
  "Takes a config, downloads the rss feed, finds the most recent file,
   downloads it, tags it, and moves it where it needs to go."
  [{:keys [name feed-url filename tag dest-dir] :as config}]
  (log/info "Beginning " name)
  (let [tempfile  (str "/tmp/" (java.util.UUID/randomUUID))
        destfile (str dest-dir "/" filename)]
    ;; clean up old one first
    (clojure.java.io/delete-file destfile true)
    (try
      (download-latest! feed-url tempfile)
      (tag-and-copy! tempfile destfile tag)
      (log/info "Completed downloading " name)
      (catch  Exception e
        (log/error e name))
      (finally
        (clojure.java.io/delete-file tempfile true)))))

(defn schedule-local
  "Hack until moments supports local time.
   Schedule a task to run based on a Chronicle specification."
  [executor spec f]
  (let [[start & rest] (c/times-for spec (l/local-now))]
    (m/schedule-at executor start (#'me.raynes.moments/chronicle-scheduler executor f rest))))

(defn format-schedule
  [frequency]
  (->> (l/local-now)
       (c/times-for frequency)
       (map #(l/format-local-time % :mysql))
       (take 5)
       (umisc/inter-str "\n")))

(defn schedule-one!
  [{:keys [name frequency] :as config}]
  (log/info "Scheduling " name " at:\n"  (format-schedule frequency) " etc etc...")
  (schedule-local executor frequency #(do-everything! config)))

(defn schedule-all!
  [configs]
  (doseq [c configs]
    (schedule-one! c)))

(defn seed!
  "Start off with forcing all the configs to run."
  [configs]
  (doseq [c configs]
    (do-everything! c)))

Remove unnecessary let.

Remove unnecessary let.


;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.

(ns containium.systems.cassandra.alia
  "The Alia 2 implementation of the Cassandra system."
  (:require [qbits.alia :as alia]
            [containium.systems :refer (require-system Startable Stoppable)]
            [containium.systems.config :as config :refer (Config)]
            [containium.systems.cassandra :refer (Cassandra cql-statements)]
            [containium.systems.logging :as logging :refer (SystemLogger refer-logging)]))
(refer-logging)

(def ^:dynamic *consistency* nil)

(def ^:dynamic *keywordize* false)

(defn- prepare*
  [{:keys [session]} query-str]
  (alia/prepare session query-str))

(defn- do-prepared*
  [{:keys [session]} statement opts values]
  (let [args (merge {:consistency *consistency*
                     :key-fn (if (:keywordize? opts *keywordize*) keyword #_else str)}
                    opts {:values values})]
    (assert (:consistency opts) "Missing :consistency key and *consistency* not bound.")
    (alia/execute session statement args)))

(defn- has-keyspace*
  [record name]
  (let [pq (prepare* record "SELECT * FROM system.schema_keyspaces WHERE keyspace_name = ?;")]
    (not (empty? (do-prepared* record pq {:consistency :one} [name])))))

(defn- write-schema*
  [record schema-str]
  (doseq [s (cql-statements schema-str)
          :let [ps (prepare* record s)]]
    (do-prepared* record ps {:consistency :one} nil)))

(defrecord AliaCassandra [cluster session logger]
  Cassandra
  (prepare [this query-str]
    (prepare* this query-str))

  (do-prepared [this statement]
    (do-prepared* this statement nil nil))

  (do-prepared [this statement opts-values]
    (cond (sequential? opts-values) (do-prepared* this statement nil opts-values)
          (map? opts-values) (do-prepared* this statement opts-values (:values opts-values))
          :else (throw (IllegalArgumentException.
                        "Parameter opts-values must be a map or sequence."))))

  (do-prepared [this statement opts values]
    (do-prepared* this statement opts values))

  (has-keyspace? [this name]
    (has-keyspace* this name))

  (keyspaced [this name]
    (AliaCassandra. cluster (alia/connect cluster name) logger))

  (write-schema [this schema-str]
    (write-schema* this schema-str))

  Stoppable
  (stop [this]
    (info logger "Stopping Alia 2 system...")
    (alia/shutdown cluster)
    (info logger "Alia 2 system stopped.")))

(defn alia
  "Create a new AliaCassandra Startable, using the specified key to
  lookup the connection details in the Config system."
  [config-key]
  (reify Startable
    (start [_ systems]
      (let [config (config/get-config (require-system Config systems) config-key)
            logger (require-system SystemLogger systems)
            _ (info logger "Starting Alia 2 system, using config:" config)
            cluster (alia/cluster config)
            session (alia/connect cluster)]
        (info logger "Alia 2 system started.")
        (AliaCassandra. cluster session logger)))))

;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.

(ns containium.systems.cassandra.alia
  "The Alia 2 implementation of the Cassandra system."
  (:require [qbits.alia :as alia]
            [qbits.alia.policy.reconnection :refer (constant-reconnection-policy)]
            [containium.systems :refer (require-system Startable Stoppable)]
            [containium.systems.config :as config :refer (Config)]
            [containium.systems.cassandra :refer (Cassandra cql-statements)]
            [containium.systems.logging :as logging :refer (SystemLogger refer-logging)]))
(refer-logging)

(def ^:dynamic *consistency* nil)

(def ^:dynamic *keywordize* false)

(defn- prepare*
  [{:keys [session]} query-str]
  (alia/prepare session query-str))

(defn- do-prepared*
  [{:keys [session]} statement opts values]
  (let [args (merge {:consistency *consistency*
                     :key-fn (if (:keywordize? opts *keywordize*) keyword #_else str)}
                    opts {:values values})]
    (assert (:consistency opts) "Missing :consistency key and *consistency* not bound.")
    (alia/execute session statement args)))

(defn- has-keyspace*
  [record name]
  (let [pq (prepare* record "SELECT * FROM system.schema_keyspaces WHERE keyspace_name = ?;")]
    (not (empty? (do-prepared* record pq {:consistency :one} [name])))))

(defn- write-schema*
  [record schema-str]
  (doseq [s (cql-statements schema-str)
          :let [ps (prepare* record s)]]
    (do-prepared* record ps {:consistency :one} nil)))

(defrecord AliaCassandra [cluster session logger]
  Cassandra
  (prepare [this query-str]
    (prepare* this query-str))

  (do-prepared [this statement]
    (do-prepared* this statement nil nil))

  (do-prepared [this statement opts-values]
    (cond (sequential? opts-values) (do-prepared* this statement nil opts-values)
          (map? opts-values) (do-prepared* this statement opts-values (:values opts-values))
          :else (throw (IllegalArgumentException.
                        "Parameter opts-values must be a map or sequence."))))

  (do-prepared [this statement opts values]
    (do-prepared* this statement opts values))

  (has-keyspace? [this name]
    (has-keyspace* this name))

  (keyspaced [this name]
    (AliaCassandra. cluster (alia/connect cluster name) logger))

  (write-schema [this schema-str]
    (write-schema* this schema-str))

  Stoppable
  (stop [this]
    (info logger "Stopping Alia 2 system...")
    (alia/shutdown cluster)
    (info logger "Alia 2 system stopped.")))

(defn alia
  "Create a new AliaCassandra Startable, using the specified key to
  lookup the connection details in the Config system."
  [config-key]
  (reify Startable
    (start [_ systems]
      (let [config (config/get-config (require-system Config systems) config-key)
            config (if-not (:reconnection-policy config)
                     (assoc config :reconnection-policy (constant-reconnection-policy 5))
                    ;else
                     config)
            logger (require-system SystemLogger systems)
            _ (info logger "Starting Alia 2 system, using config:" config)
            cluster (alia/cluster config)
            session (alia/connect cluster)]
        (info logger "Alia 2 system started.")
        (AliaCassandra. cluster session logger)))))

Alia: Set a 5 seconds reconnection policy by default