Smarter query compiler

Smarter query compiler

Rather than compiling each "=" operation independently and joining them
with UNION/INTERSECT/JOIN, we now build a list of required tables, and
WHERE clauses. "and"/"or" operations simply combine their child WHERE
clauses with AND/OR respectively. "not" will still evaluate its children
as an "or" operation, and will then wrap that clause in NOT.

This removes all unnecessary set operations and joins, and also
unnecessary subqueries, resulting in tremendous performance gains across
the board.

The only subqueries we produce now are EXISTS queries for
resource_params queries, to avoid joining n times to resource_params in
the case of "and" operations.

--- src/com/puppetlabs/puppetdb/query/resource.clj
+++ src/com/puppetlabs/puppetdb/query/resource.clj
@@ -1,19 +1,44 @@
 ;; ## SQL query compiler

 (ns com.puppetlabs.puppetdb.query.resource
-  (:refer-clojure :exclude [case compile conj! distinct disj! drop sort take])
   (:require [com.puppetlabs.utils :as utils]
             [cheshire.core :as json]
             [clojure.string :as string])
-  (:use clojureql.core
-        [com.puppetlabs.jdbc :only [query-to-vec convert-result-arrays with-transacted-connection]]
+  (:use [com.puppetlabs.jdbc :only [query-to-vec convert-result-arrays with-transacted-connection]]
         [com.puppetlabs.puppetdb.scf.storage :only [db-serialize sql-array-query-string]]
         [clojure.core.match :only [match]]))

-(defmulti compile-query->sql
-  "Recursively compile a query into a collection of SQL operations."
+(defmulti compile-term
+  "Recursively compile a query into a structured map reflecting the terms of
+  the query."
   (fn [query]
     (string/lower-case (first query))))
+
+(defn build-join-expr
+  "Builds an inner join expression between catalog_resources and the given
+  `table`. The only acceptable tables are `certnames` and `certname_catalogs`,
+  each of which *must* have rows corresponding to the `catalog_resources`
+  table. Thus this operation is safe."
+  [table]
+  (condp = table
+    ;; We will always also join to certname_catalogs if we're joining to
+    ;; certnames, but handle it separately, because we could ONLY join to
+    ;; certname_catalogs, and don't want duplicate joins. I wish this were more
+    ;; generic.
+    :certnames
+    "INNER JOIN certnames ON certname_catalogs.certname = certnames.name"
+
+    :certname_catalogs
+    "INNER JOIN certname_catalogs USING(catalog)"))
+
+(defn compile-query->sql
+  "Compile a query into a collection of SQL operations."
+  [query]
+  (let [{:keys [where joins params]} (compile-term query)
+        join-expr (->> joins
+                    (map build-join-expr)
+                    (string/join " "))]
+    (apply vector (format "SELECT catalog_resources.catalog,catalog_resources.resource FROM catalog_resources %s WHERE %s" join-expr where) params)))

 (defn query->sql
   "Compile a vector-structured query into an SQL expression.
@@ -32,15 +57,15 @@
 and their parameters which match."
   [[sql & params]]
   {:pre [(string? sql)]}
-  (let [query         (str "SELECT certname_catalogs.certname, cr.resource, cr.type, cr.title,"
-                           "cr.tags, cr.exported, cr.sourcefile, cr.sourceline, rp.name, rp.value "
-                           "FROM catalog_resources cr "
-                           "JOIN certname_catalogs USING(catalog) "
-                           "LEFT OUTER JOIN resource_params rp "
-                           "ON cr.resource = rp.resource "
-                           "WHERE (cr.catalog,cr.resource) IN "
-                           sql)
-        results       (apply query-to-vec query params)
+  (let [query         (format (str "SELECT certname_catalogs.certname, cr.resource, cr.type, cr.title,"
+                                   "cr.tags, cr.exported, cr.sourcefile, cr.sourceline, rp.name, rp.value "
+                                   "FROM catalog_resources cr "
+                                   "JOIN certname_catalogs USING(catalog) "
+                                   "LEFT OUTER JOIN resource_params rp "
+                                   "ON cr.resource = rp.resource "
+                                   "WHERE (cr.catalog,cr.resource) IN (%s)")
+                              sql)
+        results (apply query-to-vec query params)
         metadata_cols [:certname :resource :type :title :tags :exported :sourcefile :sourceline]
         metadata      (apply juxt metadata_cols)]
     (vec (for [[resource params] (group-by metadata results)]
@@ -52,56 +77,44 @@
 ;; will produce a query that selects a set of hashes matching the
 ;; predicate, which can then be combined with connectives to build
 ;; complex queries.
-(defmethod compile-query->sql "="
+(defmethod compile-term "="
   [[op path value :as term]]
   (let [count (count term)]
     (if (not (= 3 count))
       (throw (IllegalArgumentException.
               (format "%s requires exactly two arguments, but we found %d" op (dec count))))))
-  (let [catalog_resources (-> (table :catalog_resources)
-                            (project [:catalog_resources.catalog :catalog_resources.resource]))
-        tbl               (match [path]
-                                 ;; tag join.
-                                 ["tag"]
-                   [(format "SELECT catalog,resource FROM catalog_resources WHERE %s"
-                                          (sql-array-query-string "tags"))
-                                  value]
-                                 ;; node join.
-                                 [["node" "name"]]
-                                 (let [certname_catalogs (-> (table :certname_catalogs)
-                                                             (select (where
-                                                                      (= :certname_catalogs.certname value)))
-                                             (project []))]
-                                   (join certname_catalogs catalog_resources :catalog))
-                                 ;; {in,}active nodes.
-                                 [["node" "active"]]
-                                 (let [certname_catalogs (-> (table :certname_catalogs)
-                                                             (join (table :certnames)
-                                                                   (where (= :certname_catalogs.certname
-                                                                             :certnames.name)))
-                                                             (select (where (if value
-                                                                              (= :certnames.deactivated nil)
-                                                                              (not (= :certnames.deactivated nil)))))
-                                             (project []))]
-                                   (join certname_catalogs catalog_resources :catalog))
-                                 ;; param joins.
-                                 [["parameter" (name :when string?)]]
-                                 (let [resource_params (-> (table :resource_params)
-                                                           (select (where
-                                                                    (and (= :resource_params.name name)
-                                                                         (= :resource_params.value (db-serialize value)))))
-                                           (project [:resource]))
-                         [sql & params]  (compile resource_params nil)]
-                     (apply vector (format "SELECT lhs.catalog,lhs.resource FROM catalog_resources lhs WHERE EXISTS (SELECT * FROM (%s) rhs WHERE lhs.resource = rhs.resource)" sql) params))
-                                 ;; metadata match.
-                                 [(metadata :when string?)]
-                                 (select catalog_resources
-                                         (where (= (keyword metadata) value)))
-                                 ;; ...else, failure
-                                 :else (throw (IllegalArgumentException.
-                                               (str term " is not a valid query term"))))
-        [sql & params]    (if (table? tbl) (compile tbl nil) tbl)]
-    (apply vector (format "(%s)" sql) params)))
+  (match [path]
+         ;; tag join.
+         ["tag"]
+         {:where (sql-array-query-string "tags")
+          :params [value]}
+
+         ;; node join.
+         [["node" "name"]]
+         {:joins [:certname_catalogs]
+          :where "certname_catalogs.certname = ?"
+          :params [value]}
+
+         ;; {in,}active nodes.
+         [["node" "active"]]
+         {:joins [:certname_catalogs :certnames]
+          :where (format "certnames.deactivated IS %s" (if value "NULL" "NOT NULL"))}
+
+         ;; param joins.
+         [["parameter" (name :when string?)]]
+         {:where "EXISTS (SELECT rp.resource FROM resource_params rp WHERE rp.name = ? AND rp.value = ? AND rp.resource = catalog_resources.resource)"
+          :params [name (db-serialize value)]}
+
+         ;; metadata match.
+         [(metadata :when string?)]
+         (if (re-matches #"(?i)[a-z_][a-z0-9_]*" metadata)
+           {:where (format "catalog_resources.%s = ?" metadata)
+            :params [value]}
+           (throw (IllegalArgumentException. "illegal metadata column name %s" metadata)))
+
+         ;; ...else, failure
+         :else (throw (IllegalArgumentException.
+                        (str term " is not a valid query term")))))

 (defn- alias-subqueries
   "Produce distinct aliases for a list of queries, suitable for a join
@@ -112,50 +125,50 @@

 ;; Join a set of predicates together with an 'and' relationship,
 ;; performing an intersection (via natural join).
-(defmethod compile-query->sql "and"
+(defmethod compile-term "and"
   [[op & terms]]
   {:pre  [(every? vector? terms)]
-   :post [(string? (first %))
-          (every? (complement coll?) (rest %))]}
+   :post [(string? (:where %))]}
   (when (empty? terms)
     (throw (IllegalArgumentException. (str op " requires at least one term"))))
-  (let [terms  (map compile-query->sql terms)
-        params (mapcat rest terms)
-        query  (->> (map first terms)
-                   (alias-subqueries)
-                   (string/join " NATURAL JOIN ")
-                   (str "SELECT catalog,resource FROM ")
-                    (format "(%s)"))]
-    (apply vector query params)))
+  (let [terms (map compile-term terms)
+        joins (distinct (mapcat :joins terms))
+        params (mapcat :params terms)
+        query (->> (map :where terms)
+                   (map #(format "(%s)" %))
+                   (string/join " AND "))]
+    {:joins joins
+     :where query
+     :params params}))

 ;; Join a set of predicates together with an 'or' relationship,
 ;; performing a union operation.
-(defmethod compile-query->sql "or"
+(defmethod compile-term "or"
   [[op & terms]]
   {:pre  [(every? vector? terms)]
-   :post [(string? (first %))
-          (every? (complement coll?) (rest %))]}
+   :post [(string? (:where %))]}
   (when (empty? terms)
     (throw (IllegalArgumentException. (str op " requires at least one term"))))
-  (let [terms  (map compile-query->sql terms)
-        params (mapcat rest terms)
-        query  (->> (map first terms)
-                   (string/join " UNION ALL ")
-                    (format "(%s)"))]
-    (apply vector query params)))
+  (let [terms (map compile-term terms)
+        joins (distinct (mapcat :joins terms))
+        params (mapcat :params terms)
+        query (->> (map :where terms)
+                   (map #(format "(%s)" %))
+                   (string/join " OR "))]
+    {:joins joins
+     :where query
+     :params params}))

 ;; Join a set of predicates together with a 'not' relationship,
 ;; performing a set difference. This will reject resources matching
 ;; _any_ child predicate.
-(defmethod compile-query->sql "not"
+(defmethod compile-term "not"
   [[op & terms]]
   {:pre  [(every? vector? terms)]
-   :post [(string? (first %))
-          (every? (complement coll?) (rest %))]}
+   :post [(string? (:where %))]}
   (when (empty? terms)
     (throw (IllegalArgumentException. (str op " requires at least one term"))))
-  (let [[subquery & params] (compile-query->sql (cons "or" terms))
-        query               (->> subquery
-                 (format "SELECT lhs.catalog,lhs.resource FROM catalog_resources lhs WHERE NOT EXISTS (SELECT * FROM %s rhs WHERE lhs.resource = rhs.resource AND lhs.catalog = rhs.catalog)")
-                 (format "(%s)"))]
-    (apply vector query params)))
+  (let [term (compile-term (cons "or" terms))
+        query (->> (:where term)
+                   (format "NOT (%s)"))]
+    (assoc term :where query)))


Fix group verification to check access, not admin

Any user with access to a group can deploy, so that's good enough for
verification. We use admin only for managing group membership.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/tobias/clojars-web.git/'


support display of "general" errors in team form

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/clojurecup/cc2014-frontend.git/'


test(statement): remove duplicated test (#109)

Co-authored-by: Rafael Cupello <54cb9908ac4c5e67b844bebe55aea2e6d5bd2078@gmail.com>

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/B2W-Digital/pdg-core.git/'


Copy in partner-http-api tests

Copy in partner-http-api tests

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/turbovote-http-api.git/'


Add color namespace

Add color namespace

error: pathspec 'src/main/clojure/com/stuartsierra/lazytest/color.clj' did not match any file(s) known to git

--- src/main/clojure/com/stuartsierra/lazytest/color.clj
+++ src/main/clojure/com/stuartsierra/lazytest/color.clj
@@ -0,0 +1,63 @@
+(ns com.stuartsierra.lazytest.color)
+
+;; Mostly stolen from Stuart Halloway's circumspec
+
+(defn colorize?
+  "Colorize output, true if system property
+  com.stuartsierra.lazytest.colorize is true (default)"
+  [] (contains? #{"yes" "true"}
+                (System/getProperty
+                 "com.stuartsierra.lazytest.colorize"
+                 "true")))
+
+(defn set-colorize
+  "Set the colorize? property to true or false."
+  [bool]
+  (assert (instance? Boolean bool))
+  (System/setProperty "com.stuartsierra.lazytest.colorize"
+                      (str bool)))
+
+(def #^{:doc "ANSI color code table"}
+     color-table
+     {:reset "[0m"
+      :bold-on "[1m"
+      :italic-on "[3m"
+      :underline-on "[4m"
+      :inverse-on "[7m"
+      :strikethrough-on "[9m"
+      :bold-off "[22m"
+      :italic-off "[23m"
+      :underline-off "[24m"
+      :inverse-off "[27m"
+      :strikethrough-off "[29m"
+      :fg-black "[30m"
+      :fg-red "[31m"
+      :fg-green "[32m"
+      :fg-yellow "[33m"
+      :fg-blue "[34m"
+      :fg-magenta "[35m"
+      :fg-cyan "[36m"
+      :fg-white "[37m"
+      :fg-default "[39m"
+      :bg-black "[40m"
+      :bg-red "[41m"
+      :bg-green "[42m"
+      :bg-yellow "[43m"
+      :bg-blue "[44m"
+      :bg-magenta "[45m"
+      :bg-cyan "[46m"
+      :bg-white "[47m"
+      :bg-default "[49m"})
+
+(defn ansi-color-str
+  "Return ANSI color codes for the given sequence of colors, which are
+  keywords in color-table."
+  [& colors]
+  (apply str (map (fn [c] (str (char 27) (color-table c))) colors)))
+
+(defn colorize
+  "Wrap string s in ANSI colors if colorize? is true."
+  [s & colors]
+  (if (and (colorize) (seq s))
+    (str (ansi-color-string colors) (ansi-color-string :reset))
+    s))


include test database when running tests

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/vincentstorme/clojureme.git/'


Fix export test

Fix export test

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/election-authority-works.git/'


Remove ns references

Remove ns references


add write-wav multimethod for writing wav files

error: pathspec 'src/overtone/helpers/audio_file.clj' did not match any file(s) known to git

--- src/overtone/helpers/audio_file.clj
+++ src/overtone/helpers/audio_file.clj
@@ -0,0 +1,75 @@
+(ns
+    ^{:doc "Audio file encoding functions"
+      :author "Sam Aaron"}
+  overtone.helpers.audio-file
+  (:use [overtone.sc.buffer :only [buffer-data]])
+  (:import [javax.sound.sampled AudioFormat AudioFileFormat AudioFormat$Encoding
+                                AudioFileFormat$Type AudioInputStream AudioSystem]
+           [java.io File ByteArrayInputStream]
+           [java.nio ByteBuffer]))
+
+(defn- fill-data-buffer!
+  [^ByteBuffer b-data data sample-bytes]
+  (loop [idx 0
+         data data]
+    (when-not (empty? data)
+      (let [b-idx (* sample-bytes idx)]
+        (.putShort b-data b-idx (short (* (first data) Short/MAX_VALUE))))
+      (recur (inc idx) (rest data))))
+
+  b-data)
+
+(defn- write-audio-file-from-seq
+  "Writes contents of data to a new wav file with path. Adds frame-rate and
+  n-channels as file metadata for appropriate playback/consumption of the new
+  audio file."
+  [data path frame-rate n-channels]
+  (when (some #(or (< % -1) (> % 1)) data)
+    (throw (Exception. (str "Unable to write audio file with this data as it contains sample points either less than -1 or greater than 1."))))
+  (let [frame-rate   (float frame-rate)
+        n-channels   (int n-channels)
+        sample-bytes (/ Short/SIZE 8)
+        frame-bytes  (* sample-bytes n-channels)
+        a-format     (AudioFormat. AudioFormat$Encoding/PCM_SIGNED
+                                   frame-rate
+                                   Short/SIZE
+                                   n-channels
+                                   frame-bytes
+                                   frame-rate
+                                   true)
+        data-size    (count data)
+        n-bytes      (* data-size sample-bytes)
+        b-data       (ByteBuffer/allocate n-bytes)
+        b-data       (fill-data-buffer! b-data data sample-bytes)
+        stream       (AudioInputStream. (ByteArrayInputStream. (.array b-data))
+                                        a-format
+                                        data-size)
+        f            (File. path)
+        f-type       AudioFileFormat$Type/WAVE]
+    (AudioSystem/write stream f-type f)))
+
+(defn- resolve-data-type
+  [& args]
+  (let [data (first args)]
+    (cond
+     (= :overtone.sc.buffer/buffer (type data)) ::buffer
+     (sequential? data) ::sequence)))
+
+
+(defmulti write-wav
+  "Write data as a wav file. Accepts either a buffer or a sequence of values.
+  When passing a sequence, you also need to specify the frame-rate and n-channels.
+  For both, you need to pass the path of the new file as the 2nd arg.
+
+  Required args:
+  buffer [data path]
+  seq    [data path frame-rate n-channels]"
+  resolve-data-type)
+
+(defmethod write-wav ::buffer
+  [data path]
+  (write-audio-file-from-seq (buffer-data data) path (:rate data) (:n-channels data)))
+
+(defmethod write-wav ::sequence
+  [data path frame-rate n-channels]
+  (write-audio-file-from-seq (buffer-data data) path frame-rate n-channels))


Add (multiple) daemon(s) to the ns via the daemon/define macro

error: pathspec 'src/cider_ci/utils/daemon.clj' did not match any file(s) known to git

--- src/cider_ci/utils/daemon.clj
+++ src/cider_ci/utils/daemon.clj
@@ -0,0 +1,42 @@
+; Copyright (C) 2013, 2014 Dr. Thomas Schank  (DrTom@schank.ch, Thomas.Schank@algocon.ch)
+; Licensed under the terms of the GNU Affero General Public License v3.
+; See the "LICENSE.txt" file provided with this software.
+
+(ns cider-ci.utils.daemon
+  (:require
+    [clj-logging-config.log4j :as logging-config]
+    [clojure.tools.logging :as logging]
+    [cider-ci.utils.with :as with]
+    ))
+
+
+(defmacro define [daemon-name start-fn stop-fn secs-pause & body]
+  (let [stop (gensym "_stop_")]
+    `(do
+
+       (defonce ~stop (atom (fn [])))
+
+       (defn ~stop-fn []
+         (@~stop))
+
+       (defn ~start-fn []
+         (~stop-fn)
+         (let [done# (atom false)
+               runner# (future (logging/debug "daemon " ~daemon-name " started")
+                               ;(Thread/sleep (* ~secs-pause 1000))
+                               (loop []
+                                 (when-not @done#
+                                   (with/suppress-and-log-error
+                                     ~@body)
+                                   (Thread/sleep (* ~secs-pause 1000))
+                                   (recur))))]
+           (reset! ~stop (fn []
+                           (reset! done# true)
+                           (future-cancel runner#)
+                           ;@runner#
+                           (logging/debug "daemon " ~daemon-name "stopped")
+                           ))))
+       )))
+
+;(macroexpand-1 '(define "Blah" start stop 10 (logging/info "looping ...")))
+


Solve Average 1 in clojure

Solve Average 1 in clojure

error: pathspec 'solutions/uri/1005/1005.clj' did not match any file(s) known to git

--- solutions/uri/1005/1005.clj
+++ solutions/uri/1005/1005.clj
@@ -0,0 +1,8 @@
+(ns main)
+
+(defn main []
+  (let [a (Double/parseDouble (read-line))
+        b (Double/parseDouble (read-line))]
+    (format "%.5f" (/ (+ (* a 3.5) (* b 7.5)) 11.0))))
+
+(println "MEDIA =" (main))


run lein plugin in project's jvm, not lein's

--- clojure/src/main/clj/leiningen/cuke.clj
+++ clojure/src/main/clj/leiningen/cuke.clj
@@ -1,10 +1,13 @@
-(ns leiningen.cuke)
+(ns leiningen.cuke
+  (:require [leiningen.compile :as lc]))

 (defn cuke
   "runs cucumber"
   [project]
   ;; main calls System/exit kind of gross
-  (cucumber.cli.Main/main
-   (into-array String
-               ["--format" "pretty"
-                "--glue" "resources/" "resources/features"])))
+  (lc/eval-in-project
+   project
+   `(cucumber.cli.Main/main
+     (into-array String
+                 ["--format" "pretty"
+                  "--glue" "test/cucumber" "test/cucumber/features"]))))


Add login controller

Add login controller

error: pathspec 'src/madouc/controllers.clj' did not match any file(s) known to git

--- src/madouc/controllers.clj
+++ src/madouc/controllers.clj
@@ -0,0 +1,16 @@
+(ns madouc.controllers
+  (:require [ring.util.http-response :as http-resp]
+            [taoensso.timbre :as log]
+            [madouc.services.login :refer [valid-user?]]
+            [madouc.services.token :as token]
+            [clj-time.core :as t]))
+
+(defn login-controller
+  [login-req]
+  (let [username (:username login-req)
+        password (:password login-req)]
+    (if (valid-user? username password)
+      (http-resp/ok {:token (token/new username (t/now))})
+      (http-resp/unauthorized {:error "Unauthorized"}))))
+
+


more informative logging in supervisor (ported from 0.7.3)

--- src/clj/backtype/storm/daemon/supervisor.clj
+++ src/clj/backtype/storm/daemon/supervisor.clj
@@ -89,10 +89,9 @@

 (defn read-allocated-workers
   "Returns map from worker id to worker heartbeat. if the heartbeat is nil, then the worker is dead (timed out or never wrote heartbeat)"
-  [supervisor assigned-executors]
+  [supervisor assigned-executors now]
   (let [conf (:conf supervisor)
         ^LocalState local-state (:local-state supervisor)
-        now (current-time-secs)
         id->heartbeat (read-worker-heartbeats conf)
         approved-ids (set (keys (.get local-state LS-APPROVED-WORKERS)))]
     (into
@@ -188,7 +187,8 @@
   (let [conf (:conf supervisor)
         ^LocalState local-state (:local-state supervisor)
         assigned-executors (defaulted (.get local-state LS-LOCAL-ASSIGNMENTS) {})
-        allocated (read-allocated-workers supervisor assigned-executors)
+        now (current-time-secs)
+        allocated (read-allocated-workers supervisor assigned-executors now)
         keepers (filter-val
                  (fn [[state _]] (= state :valid))
                  allocated)
@@ -216,6 +216,7 @@
       (when (not= :valid state)
         (log-message
          "Shutting down and clearing state for id " id
+         ". Current supervisor time: " now
          ". State: " state
          ", Heartbeat: " (pr-str heartbeat))
         (shutdown-worker supervisor id)


Point failure messages to the correct FAQ entries

--- src/clojars/routes/repo.clj
+++ src/clojars/routes/repo.clj
@@ -148,7 +148,7 @@
   (when (and (not (snapshot-version? version))
           (storage/path-exists? storage
             (str/join "/" [(fu/group->path group-id) artifact-id version])))
-    (throw-invalid "redeploying non-snapshots is not allowed (see http://git.io/vO2Tg)")))
+    (throw-invalid "redeploying non-snapshots is not allowed (see https://git.io/v1IAs)")))

 (defn assert-jar-uploaded [artifacts pom]
   (when (and (= :jar (:packaging pom))
@@ -185,11 +185,11 @@
     ;; OSes, URLs and tools.
     (validate-regex name #"^[a-z0-9_.-]+$"
       (str "project names must consist solely of lowercase "
-        "letters, numbers, hyphens and underscores (see http://git.io/vO2Uy)"))
+        "letters, numbers, hyphens and underscores (see https://git.io/v1IAl)"))

     (validate-regex group #"^[a-z0-9_.-]+$"
       (str "group names must consist solely of lowercase "
-        "letters, numbers, hyphens and underscores (see http://git.io/vO2Uy)"))
+        "letters, numbers, hyphens and underscores (see https://git.io/v1IAl)"))

     ;; Maven's pretty accepting of version numbers, but so far in 2.5 years
     ;; bar one broken non-ascii exception only these characters have been used.
@@ -198,7 +198,7 @@
     ;; compatible for everyone let's lock it down.
     (validate-regex version #"^[a-zA-Z0-9_.+-]+$"
       (str "version strings must consist solely of letters, "
-        "numbers, dots, pluses, hyphens and underscores (see http://git.io/vO2TO)")))
+        "numbers, dots, pluses, hyphens and underscores (see https://git.io/v1IA2)")))

 (defn validate-deploy [storage dir pom {:keys [group name version]}]
   (validate-gav group name version)


add test to ensure functionality in EDT

add test to ensure functionality in EDT

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/electorate-works.git/'


New route

New route

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/aarkerio/cmsclj.git/'


Add a piece for hypothetically logging in, later

error: pathspec 'src/frereth_server/login.clj' did not match any file(s) known to git

--- src/frereth_server/login.clj
+++ src/frereth_server/login.clj
@@ -0,0 +1,5 @@
+(ns frereth-server.login
+  (:gen-class))
+
+;;;; Handle login-style requests.
+;;;; This entire idea seems more than a little dumb.


Problem 0087.

Problem 0087.

error: pathspec 'src/project_euler/problem_0087.clj' did not match any file(s) known to git

--- src/project_euler/problem_0087.clj
+++ src/project_euler/problem_0087.clj
@@ -0,0 +1,49 @@
+(ns project-euler.problem_0087)
+
+;; The smallest number expressible as the sum of a prime square, prime cube, and prime fourth power is 28.
+;; In fact, there are exactly four numbers below fifty that can be expressed in such a way:
+
+;; 28 = 2^2 + 2^3 + 2^4
+;; 33 = 3^2 + 2^3 + 2^4
+;; 49 = 5^2 + 2^3 + 2^4
+;; 47 = 2^2 + 3^3 + 2^4
+
+;; How many numbers below fifty million can be expressed as the sum of a prime square, prime cube,
+;; and prime fourth power?
+
+;; From problem 0010 :
+;; ---------------------
+(defn prime?
+  [n]
+  {:pre [(integer? n) (> n 0)]}
+  (or (== 2 n)
+      (let [mmax (+ (Math/sqrt n) 1)]
+        (not (some #(== 0 (rem n %)) (concat [2] (range 3 mmax 2))))
+        )))
+
+;;==========================================================================
+
+(defn primes-under [x]
+  (filter prime? (range 2 x)))
+
+(defn solution-for [m]
+  (let [primes (primes-under m)]
+    (->> primes
+         (map (fn [x] (* x x x x)))
+         (take-while (fn [x4] (< x4 m)))
+         (mapcat (fn [x4] (->> primes
+                               (map (fn [y] (+ x4 (* y y y))))
+                               (take-while (fn [x4+y3] (< x4+y3 m)))
+                               )))
+         (mapcat (fn [x4+y3] (->> primes
+                                  (map (fn [z] (+ x4+y3 (* z z))))
+                                  (take-while (fn [x4+y3+z2] (< x4+y3+z2 m)))
+                                  )))
+         (set)
+         (count)
+         )))
+
+(defn solution []
+  (solution-for 50000000))
+
+;;-> 1097343


Updating: initiating libatscc2clj

Updating: initiating libatscc2clj

error: pathspec 'contrib/libatscc/libatscc2clj/CATS/integer_cats.clj' did not match any file(s) known to git

--- contrib/libatscc/libatscc2clj/CATS/integer_cats.clj
+++ contrib/libatscc/libatscc2clj/CATS/integer_cats.clj
@@ -0,0 +1,142 @@
+;;
+;;;;;;
+;
+; HX-2016-07:
+; for Clojure code
+; translated from ATS
+;
+;;;;;;
+;;
+
+;;
+;;;;;;
+; beg of [integer_cats.clj]
+;;;;;;
+;;
+
+;; ****** ****** ;;
+;;
+;; HX: for signed integers
+;;
+;; ****** ****** ;;
+;;
+(defmacro
+ ats2cljpre_neg_int0[x] `(- ~x)
+)
+(defmacro
+ ats2cljpre_neg_int1[x] `(- ~x)
+)
+;;
+;; ****** ****** ;;
+;;
+(defmacro
+ ats2cljpre_abs_int0[x]
+ `(let [x# ~x] (if (>= x# 0) x# (- x#)))
+)
+(defmacro
+ ats2cljpre_abs_int1[x]
+ `(let [x# ~x] (if (>= x# 0) x# (- x#)))
+)
+;;
+;; ****** ****** ;;
+;;
+(defmacro
+ ats2cljpre_succ_int0[x] `(+ ~x 1)
+)
+(defmacro
+ ats2cljpre_pred_int0[x] `(- ~x 1)
+)
+;;
+(defmacro
+ ats2cljpre_succ_int1[x] `(+ ~x 1)
+)
+(defmacro
+ ats2cljpre_pred_int1[x] `(- ~x 1)
+)
+;;
+;; ****** ****** ;;
+;;
+(defmacro
+ ats2cljpre_add_int0_int0[x y] `(+ ~x ~y)
+)
+(defmacro
+ ats2cljpre_sub_int0_int0[x y] `(- ~x ~y)
+)
+(defmacro
+ ats2cljpre_mul_int0_int0[x y] `(* ~x ~y)
+)
+(defmacro
+ ats2cljpre_div_int0_int0[x y] `(quot ~x ~y)
+)
+(defmacro
+ ats2cljpre_mod_int0_int0[x y] `(mod ~x ~y)
+)
+(defmacro
+ ats2cljpre_rem_int0_int0[x y] `(rem ~x ~y)
+)
+;;
+(defmacro
+ ats2cljpre_add_int1_int1[x y] `(+ ~x ~y)
+)
+(defmacro
+ ats2cljpre_sub_int1_int1[x y] `(- ~x ~y)
+)
+(defmacro
+ ats2cljpre_mul_int1_int1[x y] `(* ~x ~y)
+)
+(defmacro
+ ats2cljpre_div_int1_int1[x y] `(quot ~x ~y)
+)
+(defmacro
+ ats2cljpre_mod_int1_int1[x y] `(mod ~x ~y)
+)
+(defmacro
+ ats2cljpre_rem_int1_int1[x y] `(rem ~x ~y)
+)
+;;
+;; ****** ****** ;;
+;;
+(defmacro
+ ats2cljpre_lt_int0_int0[x y] `(< ~x ~y)
+)
+(defmacro
+ ats2cljpre_lte_int0_int0[x y] `(<= ~x ~y)
+)
+(defmacro
+ (ats2cljpre_gt_int0_int0[x y] `(> ~x ~y)
+)
+(defmacro
+ (ats2cljpre_gte_int0_int0[x y] `(>= ~x ~y)
+)
+;;
+(defmacro
+ ats2cljpre_eq_int0_int0[x y] `(= ~x ~y)
+)
+(defmacro
+ ats2cljpre_neq_int0_int0[x y] `(not (= ~x ~y))
+)
+;;
+;;
+(defmacro
+ ats2cljpre_lt_int1_int1[x y] `(< ~x ~y)
+)
+(defmacro
+ ats2cljpre_lte_int1_int1[x y] `(<= ~x ~y)
+)
+(defmacro
+ ats2cljpre_gt_int1_int1[x y] `(> ~x ~y)
+)
+(defmacro
+ ats2cljpre_gte_int1_int1[x y] `(>= ~x ~y)
+)
+;;
+(defmacro
+ ats2cljpre_eq_int1_int1[x y] `(= ~x ~y)
+)
+(defmacro
+ ats2cljpre_neq_int1_int1[x y] `(not (= ~x ~y))
+)
+;;
+;; ****** ****** ;;
+
+;; end of [integer_cats.clj] ;;


Only spawn a thread for cache misses

Only spawn a thread for cache misses


Disable /index.html test

Disable /index.html test

…because wtf!?!

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/physion/ovation-webapi.git/'


adds notification for user feedback on deletion

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/starcity-properties/odin.git/'


evidence tests

evidence tests

error: pathspec 'test/clj/salava/badge/evidence_test.clj' did not match any file(s) known to git

--- test/clj/salava/badge/evidence_test.clj
+++ test/clj/salava/badge/evidence_test.clj
@@ -0,0 +1,91 @@
+(ns salava.badge.evidence-test
+  (:require [salava.badge.main :as b]
+            [clojure.test :refer :all]
+            [salava.core.test-utils :as t]
+            [salava.core.migrator :as migrator]
+            [salava.core.util :refer [get-full-path]]))
+
+(def test-user {:id 1 :role "user" :private false :activated true})
+(def test-evidence {:name "test-evidence"
+                    :narrative "Just testing"
+                    :url "http://example.com"})
+(def badge-id 2)
+
+(t/deftest-ctx evidence-test [ctx]
+  (testing "Evidences"
+    (testing "evidence count"
+      (let [connect (b/badge-evidences ctx badge-id)]
+        (is (= 1 (count connect)))
+        ))
+    (testing "new evidence"
+      (let [id (b/save-new-evidence! ctx (:id test-user) badge-id test-evidence)]
+        (testing "get evidences"
+          (let [evidences (b/badge-evidences ctx badge-id)]
+            (is (= 2 (count evidences)))
+            (is (= (:name test-evidence) (:name (last evidences))))
+            ))
+        (testing "update evidences"
+          (let [connect (b/update-evidence! ctx badge-id id (assoc test-evidence :narrative "Testing"))
+                evidences (b/badge-evidences ctx badge-id)]
+            (is (= connect id))
+            (is (= "Testing") (:name (last evidences)))
+            ))
+        (testing "delete evidence"
+          (let [connect (b/delete-evidence! ctx id badge-id (:id test-user))
+                evidences (b/badge-evidences ctx badge-id)]
+            (is (= "success" (:status connect)))
+            (is (= 1 (count evidences)))
+            ))
+        )
+      )
+
+    (testing "evidence properties"
+      (let [id (b/save-new-evidence! ctx (:id test-user) badge-id (assoc test-evidence :resource_type "url"))]
+        (testing "evidence property"
+          (let [connect (b/badge-evidences ctx badge-id (:id test-user))
+                evidence (last connect)]
+            (is (= "url" (get-in evidence [:properties :resource_type])))
+            (is (= false (get-in evidence [:properties :hidden])))))
+
+        (testing "hide evidence"
+          (let [connect (b/toggle-show-evidence! ctx badge-id id true (:id test-user))
+                evidences (b/badge-evidences ctx badge-id (:id test-user))]
+            (is (= "success" (:status connect)))
+            (is (= true (get-in (last evidences) [:properties :hidden])))
+            ))
+        (testing "show evidence"
+          (let [connect (b/toggle-show-evidence! ctx badge-id id false (:id test-user))
+                evidences (b/badge-evidences ctx badge-id (:id test-user))]
+            (is (= "success" (:status connect)))
+            (is (= false (get-in (last evidences) [:properties :hidden])))
+            ))
+        (testing "delete evidence"
+          (let [connect (b/delete-evidence! ctx id badge-id (:id test-user))
+                evidences (b/badge-evidences ctx badge-id)]
+            (is (= "success" (:status connect)))
+            (is (= 1 (count evidences)))
+            ))
+        )
+      )
+    (testing "resource is evidence"
+      (let [full-path (get-full-path ctx)
+            page-url (str full-path "/page/view/" 1)
+            id (b/save-new-evidence! ctx (:id test-user) badge-id (assoc test-evidence :resource_type "page" :url page-url :resource_id 1))]
+        (testing "is-evidence?"
+          (let [connect (b/is-evidence? ctx (:id test-user) {:id 1 :resource_type "page"})]
+            (is (= true connect))
+            ))
+
+        (testing "delete evidence"
+          (let [connect (b/delete-evidence! ctx id badge-id (:id test-user))
+                evidences (b/badge-evidences ctx badge-id)]
+            (is (= "success" (:status connect)))
+            (is (= 1 (count evidences)))
+            ))
+
+        )
+      )
+    )
+  )
+
+(migrator/reset-seeds (migrator/test-config))


add notice-date to the transition-create resolver


replace jsonb with json (pg 9.3 compatibility)

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/fhirbase/fhirterm.git/'


Remove superfluous `->>` form

Remove superfluous `->>` form

--- src/metabase/driver/query_processor.clj
+++ src/metabase/driver/query_processor.clj
@@ -351,7 +351,7 @@
                               (filter (complement (partial contains? (set (concat breakout-kws non-breakout-kws))))))]

     ;; Create a combined sequence of aggregate result KWs + other ordered kws
-    (->> (concat breakout-kws ag-kws non-breakout-kws))))
+    (concat breakout-kws ag-kws non-breakout-kws)))

 (defn- add-fields-extra-info
   "Add `:extra_info` about `ForeignKeys` to `Fields` whose `special_type` is `:fk`."


Copy code of canonical-base implementation to c.c.algorithms

To optimize it there

Signed-off-by: Daniel Borchmann <25857343a15bf1edafebc2912e82ecf775c585c1@mailbox.tu-dresden.de>

error: pathspec 'src/conexp/contrib/algorithms/canonical_base.clj' did not match any file(s) known to git

--- src/conexp/contrib/algorithms/canonical_base.clj
+++ src/conexp/contrib/algorithms/canonical_base.clj
@@ -0,0 +1,101 @@
+;; Copyright (c) Daniel Borchmann. All rights reserved.
+;; The use and distribution terms for this software are covered by the
+;; Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+;; which can be found in the file LICENSE at the root of this distribution.
+;; By using this software in any fashion, you are agreeing to be bound by
+;; the terms of this license.
+;; You must not remove this notice, or any other, from this software.
+
+(in-ns 'conexp.contrib.algorithms)
+
+(use 'conexp.contrib.algorithms.bitwise
+     '[conexp.fca.implications :only (premise conclusion implication? make-implication)]
+     '[conexp.base :only (next-closed-set-in-family subset? forall)]
+     '[conexp.fca.contexts :only (attributes context-attribute-closure)])
+
+;;; Computing Implicational Closures
+
+(defn- add-immediate-elements
+  [implications initial-set subset-test]
+  (loop [conclusions  (transient initial-set),
+         impls        implications,
+         unused-impls (transient [])]
+    (if-let [impl (first impls)]
+      (if (subset-test (premise impl) initial-set)
+        (recur (reduce conj! conclusions (conclusion impl))
+               (rest impls)
+               unused-impls)
+        (recur conclusions
+               (rest impls)
+               (conj! unused-impls impl)))
+      [(persistent! conclusions)
+       (persistent! unused-impls)])))
+
+(defn close-under-implications
+  "Computes smallest superset of set being closed under given implications."
+  [implications set]
+  (assert (set? set))
+  (loop [set   set,
+         impls implications]
+    (let [[new impls] (add-immediate-elements impls set subset?)]
+      (if (= new set)
+        new
+        (recur new impls)))))
+
+(defn clop-by-implications
+  "Returns closure operator given by implications."
+  [implications]
+  (partial close-under-implications implications))
+
+
+;;; Canonical Base
+
+(defn canonical-base-from-clop
+  ([clop base]
+     (canonical-base-from-clop clop base #{} (constantly true)))
+  ([clop base background-knowledge]
+     (canonical-base-from-clop clop base background-knowledge (constantly true)))
+  ([clop base background-knowledge predicate]
+     (assert (fn? clop)
+             "Given closure operator must be a function")
+     (assert (coll? base)
+             "Base must be a collection")
+     (assert (fn? predicate)
+             "Predicate must be a function")
+     (assert (and (set? background-knowledge)
+                  (forall [x background-knowledge]
+                    (implication? x)))
+             "Background knowledge must be a set of implications")
+     (let [next-closure (fn [implications last]
+                          (next-closed-set-in-family predicate
+                                                     base
+                                                     (clop-by-implications implications)
+                                                     last)),
+           runner       (fn runner [implications candidate]
+                          (when candidate
+                            (let [conclusions (clop candidate)]
+                              (if (not= candidate conclusions)
+                                (let [impl  (make-implication candidate conclusions),
+                                      impls (conj implications impl)]
+                                  (cons impl
+                                        (lazy-seq (runner impls (next-closure impls candidate)))))
+                                (recur implications (next-closure implications candidate))))))]
+       (lazy-seq (runner background-knowledge
+                         (close-under-implications background-knowledge #{}))))))
+
+(defn canonical-base
+  ([ctx]
+     (canonical-base ctx #{} (constantly true)))
+  ([ctx background-knowledge]
+     (canonical-base ctx background-knowledge (constantly true)))
+  ([ctx background-knowledge predicate]
+     (assert (context? ctx)
+             "First argument must be a formal context")
+     (canonical-base-from-clop #(context-attribute-closure ctx %)
+                               (attributes ctx)
+                               background-knowledge
+                               predicate)))
+
+;;;
+
+nil


That seems to work quite reliably.

That seems to work quite reliably.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/psy-q/vinylla.git/'


Add temporary feature flag to unsubscribe-aws-handler

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/turbovote-http-api.git/'


shutdown be replaced by close

shutdown be replaced by close

--- storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
+++ storm-core/src/clj/org/apache/storm/daemon/supervisor.clj
@@ -940,8 +940,8 @@
                (.close (:heartbeat-timer supervisor))
                (.close (:event-timer supervisor))
                (.close (:blob-update-timer supervisor))
-               (.shutdown event-manager)
-               (.shutdown processes-event-manager)
+               (.close event-manager)
+               (.close processes-event-manager)
                (.shutdown (:localizer supervisor))
                (.disconnect (:storm-cluster-state supervisor)))
      SupervisorDaemon


updating names list for notifications

updating names list for notifications

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/election-notification-works.git/'


Matt's x121 solution

Matt's x121 solution

error: pathspec 'src/main/cljc/clojure_club/x121_universal_computation_engine/mgrimm.cljc' did not match any file(s) known to git


Removing unwanted rebalance topology call from nimbus_test

--- storm-core/test/clj/backtype/storm/nimbus_test.clj
+++ storm-core/test/clj/backtype/storm/nimbus_test.clj
@@ -191,7 +191,7 @@
       )))

 (deftest test-assignment
-  (with-local-cluster [cluster :supervisors 4 :ports-per-supervisor 3
+  (with-simulated-time-local-cluster [cluster :supervisors 4 :ports-per-supervisor 3
                        :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
     (let [state (:storm-cluster-state cluster)
           nimbus (:nimbus cluster)
@@ -206,9 +206,7 @@
                       "4" (thrift/mk-bolt-spec {"1" :global "2" :none} (TestPlannerBolt.) :parallelism-hint 4)}
                      )
           _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
-          ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-          _ (.rebalance nimbus "mystorm" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-          _ (Thread/sleep 1000)
+          _ (advance-cluster-time cluster 11)
           task-info (storm-component->task-info cluster "mystorm")]
       (check-consistency cluster "mystorm")
       ;; 3 should be assigned once (if it were optimized, we'd have
@@ -219,9 +217,7 @@
       (is (= 1 (count (task-info "3"))))
       (is (= 4 (storm-num-workers state "mystorm")))
       (submit-local-topology nimbus "storm2" {TOPOLOGY-WORKERS 20} topology2)
-      ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-      (.rebalance nimbus "storm2" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-      (Thread/sleep 1000)
+      (advance-cluster-time cluster 11)
       (check-consistency cluster "storm2")
       (is (= 2 (count (.assignments state nil))))
       (let [task-info (storm-component->task-info cluster "storm2")]
@@ -338,7 +334,7 @@
       )))

 (deftest test-zero-executor-or-tasks
-  (with-local-cluster [cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+  (with-simulated-time-local-cluster [cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
     (let [state (:storm-cluster-state cluster)
           nimbus (:nimbus cluster)
           topology (thrift/mk-topology
@@ -346,9 +342,7 @@
                     {"2" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 1 :conf {TOPOLOGY-TASKS 2})
                      "3" (thrift/mk-bolt-spec {"2" :none} (TestPlannerBolt.) :conf {TOPOLOGY-TASKS 5})})
           _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
-          ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-          _ (.rebalance nimbus "mystorm" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-          _ (Thread/sleep 1000)
+          _ (advance-cluster-time cluster 11)
           task-info (storm-component->task-info cluster "mystorm")]
       (check-consistency cluster "mystorm")
       (is (= 0 (count (task-info "1"))))
@@ -358,16 +352,14 @@
       )))

 (deftest test-executor-assignments
-  (with-local-cluster [cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
+  (with-simulated-time-local-cluster[cluster :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
     (let [nimbus (:nimbus cluster)
           topology (thrift/mk-topology
                     {"1" (thrift/mk-spout-spec (TestPlannerSpout. true) :parallelism-hint 3 :conf {TOPOLOGY-TASKS 5})}
                     {"2" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 8 :conf {TOPOLOGY-TASKS 2})
                      "3" (thrift/mk-bolt-spec {"2" :none} (TestPlannerBolt.) :parallelism-hint 3)})
           _ (submit-local-topology nimbus "mystorm" {TOPOLOGY-WORKERS 4} topology)
-          ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-          _ (.rebalance nimbus "mystorm" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-          _ (Thread/sleep 1000)
+          _ (advance-cluster-time cluster 11)
           task-info (storm-component->task-info cluster "mystorm")
           executor-info (->> (storm-component->executor-info cluster "mystorm")
                              (map-val #(map executor-id->tasks %)))]
@@ -383,7 +375,7 @@
       )))

 (deftest test-over-parallelism-assignment
-  (with-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
+  (with-simulated-time-local-cluster [cluster :supervisors 2 :ports-per-supervisor 5
                        :daemon-conf {SUPERVISOR-ENABLE false TOPOLOGY-ACKER-EXECUTORS 0 TOPOLOGY-EVENTLOGGER-EXECUTORS 0}]
     (let [state (:storm-cluster-state cluster)
           nimbus (:nimbus cluster)
@@ -394,9 +386,7 @@
                       "4" (thrift/mk-bolt-spec {"1" :none} (TestPlannerBolt.) :parallelism-hint 10)}
                      )
           _ (submit-local-topology nimbus "test" {TOPOLOGY-WORKERS 7} topology)
-          ;; Instead of sleeping until topology is scheduled, rebalance topology so mk-assignments is called.
-          _ (.rebalance nimbus "test" (doto (RebalanceOptions.) (.set_wait_secs 0)))
-          _ (Thread/sleep 1000)
+          _ (advance-cluster-time cluster 11)
           task-info (storm-component->task-info cluster "test")]
       (check-consistency cluster "test")
       (is (= 21 (count (task-info "1"))))


added xhr.cljs

added xhr.cljs

error: pathspec 'resources/leiningen/new/threejs_game/src/cljs/xhr.cljs' did not match any file(s) known to git

--- resources/leiningen/new/threejs_game/src/cljs/xhr.cljs
+++ resources/leiningen/new/threejs_game/src/cljs/xhr.cljs
@@ -0,0 +1,35 @@
+(ns {{project-ns}}.xhr
+  (:require [goog.net.XhrIo]))
+
+(defn send-xhr
+  "Send a xhr to url using callback and HTTP method."
+  [url callback method & [data headers timeout]]
+  (.send goog.net.XhrIo url callback method data headers timeout))
+
+(defn xhrio-wrapper
+  "A callback for processing the xhrio response event. If
+  response.target.isSuccess() is true, call f on the json response"
+  [f response]
+  (let [target (.-target response)]
+    (if (.isSuccess target)
+      (f (.getResponseJson target))
+      (.log js/console
+            (str "xhrio-wrapper error:" (aget target "lastError_"))))))
+
+(defn retrieve-url
+  "Retrieve and process json response with f from url using HTTP method and json
+  data. Optionally, define a timeout in ms."
+  [url method data f & [timeout]]
+  (let [header (clj->js {"Content-Type" "application/json"})]
+    (send-xhr url f method data header timeout)))
+
+(defn process-json
+  "Take a response, convert it a clj map and call f on the resulting map."
+  [f response]
+  (f (js->clj response :keywordize-keys true)))
+
+(defn process-json-response
+  "Assuming the server will respond with JSON, convert the response to JSON
+  and call f on it."
+  [f]
+  (partial xhrio-wrapper (partial process-json f)))


Namespace voter reg. route names

Namespace voter reg. route names


Create server.clj with Compojure

Create server.clj with Compojure

error: pathspec 'markov-clj/src/markov_clj/server.clj' did not match any file(s) known to git

--- markov-clj/src/markov_clj/server.clj
+++ markov-clj/src/markov_clj/server.clj
@@ -0,0 +1,10 @@
+(ns markov-clj.server
+  (:use compojure.core)
+  (:require [compojure.route :as route]))
+
+(defroutes app
+  (GET "/" [] "
Hello World
")
+  (route/not-found "
Page not found
"))
+
+
+(defn -main [] (start-server))


nicer formatting for the korma form in Generic SQL QP output

--- src/metabase/driver/generic_sql/query_processor.clj
+++ src/metabase/driver/generic_sql/query_processor.clj
@@ -2,6 +2,7 @@
   "The Query Processor is responsible for translating the Metabase Query Language into korma SQL forms."
   (:require [clojure.core.match :refer [match]]
             [clojure.tools.logging :as log]
+            [clojure.walk :as walk]
             [korma.core :refer :all]
             [metabase.config :as config]
             [metabase.driver.query-processor :as qp]
@@ -201,7 +202,11 @@
   [korma-form]
   (when-not qp/*disable-qp-logging*
     (log/debug
-     (u/format-color 'green "\n\nKORMA FORM:\n%s" (u/pprint-to-str korma-form))
+     (u/format-color 'green "\n\nKORMA FORM:\n%s" (->> (nth korma-form 2)                                ; korma form is wrapped in a let clause. Discard it
+                                                       (walk/prewalk (fn [form]                          ; strip korma.core/ qualifications from symbols in the form
+                                                                       (if-not (symbol? form) form       ; to remove some of the clutter
+                                                                               (symbol (name form)))))
+                                                       (u/pprint-to-str)))
      (u/format-color 'blue  "\nSQL:\n%s\n"        (eval (let [[let-form binding-form & body] korma-form] ; wrap the (select ...) form in a sql-only clause
                                                           `(~let-form ~binding-form                      ; has to go there to work correctly
-                                                             (sql-only ~@body))))))))
+                                                                      (sql-only ~@body))))))))


safe guard against bad types, validation hasn't occurred yet

TODO: note really bomb test bad data here

--- plugin/src/leiningen/figwheel.clj
+++ plugin/src/leiningen/figwheel.clj
@@ -291,10 +291,12 @@
         classpath-builds (filter #(class-path-build-ids
                                    (and (named? (:id %))
                                         (name (:id %))))
-                                 all-builds)]
+                                  (filter map? all-builds))]
     (vec
      (distinct
-      (mapcat :source-paths classpath-builds)))))
+      (apply
+       concat
+       (filter sequential? (map :source-paths classpath-builds)))))))

 (comment
   (def test-project


add view test code

add view test code

error: pathspec 'src/ca_recipe/tnywb/view.clj' did not match any file(s) known to git

--- src/ca_recipe/tnywb/view.clj
+++ src/ca_recipe/tnywb/view.clj
@@ -0,0 +1,8 @@
+(ns mbfpp.oo.tinyweb.steptwo
+  (:import (com.mblinn.oo.tinyweb RenderingException)))
+(defn test-controller [http-request] {:name (http-request :body)})
+(defn test-view [model]
+  (str "
Hello, " (model :name) "
"))
+(defn- render [view model] (try
+                             (view model)
+                             (catch Exception e (throw (RenderingException. e)))))


New module: exec

New module: exec

error: pathspec 'src/emender_service/exec.clj' did not match any file(s) known to git

--- src/emender_service/exec.clj
+++ src/emender_service/exec.clj
@@ -0,0 +1,40 @@
+(ns emender-service.exec)
+
+(defn exec
+    "Execute external command and pass given number of parameters to it."
+    ; command called without parameters
+    ([command]
+     (doto (. (Runtime/getRuntime) exec command)
+           (.waitFor)
+           (.destroy)))
+    ; command called with one parameter
+    ([command param1]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1))
+           (.waitFor)
+           (.destroy)))
+    ; command called with two parameters
+    ([command param1 param2]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1 " " param2))
+           (.waitFor)
+           (.destroy)))
+    ; command called with three parameters
+    ([command param1 param2 param3]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1 " " param2 " " param3))
+           (.waitFor)
+           (.destroy)))
+    ; command called with four parameters
+    ([command param1 param2 param3 param4]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1 " " param2 " " param3 " " param4))
+           (.waitFor)
+           (.destroy)))
+    ; command called with five parameters
+    ([command param1 param2 param3 param4 param5]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1 " " param2 " " param3 " " param4 " " param5))
+           (.waitFor)
+           (.destroy)))
+    ; command called with six parameters
+    ([command param1 param2 param3 param4 param5 param6]
+     (doto (. (Runtime/getRuntime) exec (str command " " param1 " " param2 " " param3 " " param4 " " param5 " " param6))
+           (.waitFor)
+           (.destroy))))
+


don't set term end to day prior to moveout during transition


payment is not a teller payment yet

payment is not a teller payment yet


Add controls namespace for control-events

error: pathspec 'src/cljs/gtfve/controllers/controls.cljs' did not match any file(s) known to git

--- src/cljs/gtfve/controllers/controls.cljs
+++ src/cljs/gtfve/controllers/controls.cljs
@@ -0,0 +1,14 @@
+(ns gtfve.controllers.controls
+  (:require-macros [cljs.core.async.macros :refer [go go-loop]]
+                   [gtfve.macros :refer [<?]])
+  (:require [om.core :as om]
+            [cljs.core.async :as async :refer [put! <! close!]]))
+
+(defmulti control-event
+  (fn [event args state] event))
+
+(defmethod control-event :default [_]
+  (println "Unknown control"))
+
+(defmethod control-event :side-panel-changed [_ [key] state]
+  (om/update! (:panel state) :tab key))


added regex for form validation

added regex for form validation


added clojure api for spark type creation

error: pathspec 'src/clojure/sparkling/sql/types.clj' did not match any file(s) known to git

--- src/clojure/sparkling/sql/types.clj
+++ src/clojure/sparkling/sql/types.clj
@@ -0,0 +1,66 @@
+(ns sparkling.sql.types
+  "allows for creation of dataframes from an existing rdd, steps:
+  1. turn rdd into rows using create row
+  2. create struct type passing in representation of the schema
+  3. pass rdd of rows and struct type to sparkling.sql/data-frame"
+  (:import [org.apache.spark.sql.types DataTypes]))
+
+;The list of types supported by spark
+
+(def primitive-types
+  {:string            'DataTypes/StringType
+   :binary            'DataTypes/BinaryType
+   :boolean           'DataTypes/BooleanType
+   :date              'DataTypes/DateType
+   :timestamp         'DataTypes/TimestampType
+   :calendar-interval 'DataTypes/CalendarIntervalType
+   :double            'DataTypes/DoubleType
+   :float             'DataTypes/FloatType
+   :byte              'DataTypes/ByteType
+   :integer           'DataTypes/IntegerType
+   :long              'DataTypes/LongType
+   :short             'DataTypes/ShortType
+   :null              'DataTypes/NullType})
+
+(defmacro def-primitive-data-types
+  "Macro to generate functions to create the primitive DataTypes"
+  []
+  (let [def-all-types
+        (->> primitive-types
+             (map (fn [[t v]] (list 'defn (symbol (str (name t) "-type")) [] v)))
+             (cons 'do))]
+    def-all-types))
+
+(def-primitive-data-types)
+
+(defn decimal-type
+  "create decimal type"
+  [precision scale]
+  (DataTypes/createDecimalType precision scale))
+
+(defn array-type
+  "create spark array type"
+  ([element-type nullable?] (DataTypes/createArrayType element-type nullable?))
+  ([element-type] (array-type element-type true)))
+
+(defn map-type
+  "create spark map type"
+  ([key-type value-type value-nullable?] (DataTypes/createMapType key-type value-type value-nullable?))
+  ([key-type value-type] (map-type key-type value-type true)))
+
+(defn create-row
+  "create row from a tuple of values, the values should all conform to a DataType.
+   used for creating data frame from a rdd"
+  [values]
+  (RowFactory/create (into-array Object values)))
+
+(defn- struct-field
+  "struct field is sparks internal representation of a schema for one column/attribute"
+  [{:keys [name type nullable?]}]
+  (DataTypes/createStructField name type nullable?))
+
+(defn struct-type
+  "Creates a struct type, expect each item in fields to be a map with the keys :name :type and :optional?
+  where name is a string, type is a DataType and optional is a boolean"
+  [& fields]
+  (DataTypes/createStructType (into-array (map struct-field fields))))


Add reschedule tests

Add reschedule tests

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/scheduled-message-works.git/'


somehow these dependencies weren't in the project.clj file.

--- project.clj
+++ project.clj
@@ -13,9 +13,11 @@
                  [org.clojure/core.memoize "0.5.7"]                   ; needed by core.match; has useful FIFO, LRU, etc. caching mechanisms
                  [org.clojure/data.csv "0.1.2"]                       ; CSV parsing / generation
                  [org.clojure/data.json "0.2.6"]                      ; JSON parsing / generation
+                 [org.clojure/java.classpath "0.2.2"]
                  [org.clojure/java.jdbc "0.3.6"]                      ; basic jdbc access from clojure
                  [org.clojure/tools.logging "0.3.1"]                  ; logging framework
                  [org.clojure/tools.macro "0.1.5"]                    ; tools for writing macros
+                 [org.clojure/tools.namespace "0.2.10"]
                  [org.clojure/tools.trace "0.7.8"]                    ; "tracing macros/fns to help you see what your code is doing"
                  [amalloy/ring-gzip-middleware "0.1.3"]               ; Ring middleware to GZIP responses if client can handle it
                  [cheshire "5.4.0"]                                   ; fast JSON encoding (used by Ring JSON middleware)


Actually, begin work on next pre-stable minor release.

--- project.clj
+++ project.clj
@@ -1,4 +1,4 @@
-(defproject com.damballa/abracad "0.1.1-SNAPSHOT"
+(defproject com.damballa/abracad "0.2.0-SNAPSHOT"
   :description "De/serialize Clojure data structures with Avro."
   :url "http://github.com/damballa/abracad"
   :license {:name "Eclipse Public License"


version bumps

version bumps

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/CraZySacX/scene.git/'


Completed sum-of-odd-elements.clj using reduce

error: pathspec 'hackerrank/functional_programming/clojure/introduction/sum-of-odd-elements.clj' did not match any file(s) known to git

--- hackerrank/functional_programming/clojure/introduction/sum-of-odd-elements.clj
+++ hackerrank/functional_programming/clojure/introduction/sum-of-odd-elements.clj
@@ -0,0 +1,22 @@
+
+;; Return sum of odd elements from an list. The input and output portions will be handled automatically. You need to write a function with the recommended method signature.
+;;
+;; Constraints
+;; The list will have 1-100 elements. Each element will be an integer X such that -100<=X<=100.
+;;
+;; Sample Input
+;;
+;; 3
+;; 2
+;; 4
+;; 6
+;; 5
+;; 7
+;; 8
+;; 0
+;; 1
+;; Sample Output
+;;
+;; 16
+
+(fn [lst] (reduce (fn [x y] (if (odd? y) (+ y x) x)) lst))


Add domina.xpath namespace for use in screencast

--- src/app/cljs/one/sample/view.cljs
+++ src/app/cljs/one/sample/view.cljs
@@ -1,7 +1,8 @@
 (ns ^{:doc "Render the views for the application."}
   one.sample.view
-  (:use [domina :only (xpath set-html! set-styles! styles by-id set-style!
+  (:use [domina :only (set-html! set-styles! styles by-id set-style!
                        by-class value set-value! set-text! nodes single-node)]
+        [domina.xpath :only (xpath)]
         [one.browser.animation :only (play)])
   (:require-macros [one.sample.snippets :as snippets])
   (:require [goog.events.KeyCodes :as key-codes]


extend Sedan protocol portably to various types

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/QuiltProject/sedan.git/'


Remove regex for svg images that was no longer used without webicons

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/magoyette/marcandregoyette.com.git/'


Add tree hierarchy.

Add tree hierarchy.

error: pathspec 'src/exp/simple_tree_hierarchy.clj' did not match any file(s) known to git

--- src/exp/simple_tree_hierarchy.clj
+++ src/exp/simple_tree_hierarchy.clj
@@ -0,0 +1,315 @@
+(ns simple-dag-hierarchy
+  (:require [edu.berkeley.ai.util :as util]
+            [edu.berkeley.ai.util  [graphs :as graphs] [graphviz :as gv]]
+            [exp [env :as env]  [hierarchy :as hierarchy] [sas :as sas] [sas-analysis :as sas-analysis]])
+  (:import [java.util HashMap])
+  )
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Global bindings ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;; Note: in purely static analysis, either cannot prune source vals, or must
+;; split HLAs.  Here, this amounts to, more or less, just removing outgoing arcs from goal.
+;; We can just do this implicitly, and forget about real acyclic stuff in this version.
+; TODO: (could do better, e.g., for line DTGs)
+
+
+
+(declare make-action-hla)
+
+(deftype Simple-DAG-Hierarchy
+  [sas-problem
+   predecessor-var-set
+   ancestor-var-set
+   var-levels
+   dtg-to       ; fn from var & dst-val to map from cur-val to map from nxt-val to actions
+   cycle-to     ; from from var & dst-val to map from cur-val to bool, if can cycle from cur-val
+   greedy-optimization?])
+
+(defn make-simple-dag-hierarchy [sas-problem]
+  (let [causal-graph (sas-analysis/standard-causal-graph sas-problem)
+        pred-var-set (util/map-vals (comp set #(map first %))
+                                    (util/unsorted-group-by second causal-graph))
+        dtgs         (sas-analysis/domain-transition-graphs sas-problem)
+        dtg-to       (fn [var dst-val] (util/safe-get dtgs var))]
+
+    (assert (graphs/dag? causal-graph))
+    (assert (sas-analysis/homogenous? sas-problem))
+
+    (hierarchy/SimpleHierarchicalEnv sas-problem
+      [(make-action-hla
+        (Simple-DAG-Hierarchy
+         sas-problem
+         pred-var-set
+         (util/memoized-fn ancestor-var-set [var]
+           (conj (--> (util/safe-get pred-var-set var)
+                      (map ancestor-var-set)
+                      (apply clojure.set/union))
+                 var))
+         (graphs/topological-sort-indices causal-graph)
+         dtg-to
+         (util/memoized-fn cycle-to [var dst-val]
+           (let [dtg (dtg-to var dst-val)
+                 tsi (graphs/topological-sort-indices
+                      (for [[pval emap] dtg, eval (keys emap)
+                            :when (not (= pval dst-val))]
+                        [pval eval]))]
+             (reduce (fn [m next-vars]
+                       (if (> (count next-vars) 1)
+                           (reduce #(assoc %1 %2 true) m next-vars)
+                         (let [v (util/safe-singleton next-vars)]
+                           (assoc m v (some m (keys (get dtg v)))))))
+                     {} (reverse (map #(map key %) (vals (util/group-by val tsi)))))))
+         false)
+        (->> dtgs
+             (get-in [sas/goal-var-name sas/goal-false-val sas/goal-true-val])
+             util/safe-singleton))])))
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Action Protocol ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+
+(defprotocol SDH-HLA
+  (precond-var-set [a])
+  (effect-sets     [a])
+  (pre-ref-pairs   [a])
+  (compile-hla     [a retain-type-set])
+  (extend-hla!     [a init-sets par-effect-sets])
+  (hla-type        [a]))
+
+(extend ::env/FactoredPrimitive
+  SDH-HLA
+    {:precond-var-set (fn [a] (util/keyset (:precond-map a)))
+;     :initial-sets    (fn [a] (util/map-vals (fn [x] #{x}) (:precond-map a))) ;?
+     :effect-sets     (fn [a] (util/map-vals (fn [x] #{x}) (:effect-map a)))
+     :pre-ref-pairs   (fn [a] (throw (UnsupportedOperationException.)))
+     :compile-hla     (fn [a retain-type-set] a)
+     :extend-hla!     (fn [a init-sets par-effect-sets] a)
+     :hla-type        (fn [a] (throw (UnsupportedOperationException.)))})
+
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Precond HLAs ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+(deftype SDH-Precond-HLA  [hierarchy name var dst-val dtg precond-var-set cycle-level] :as this
+  SDH-HLA
+  env/Action
+    (action-name [] name)
+    (primitive?  [] false)
+  env/ContextualAction
+    (precondition-context [s] precond-var-set)
+  hierarchy/HighLevelAction
+    (immediate-refinements- [s]
+      (let [cur-val (env/get-var s var)])
+        (if (= cur-val dst-val)
+            [[]]
+          (for [[as (vals (get dtg cur-val)), action as]]
+            [(make-action-hla action) this])))
+    (cycle-level-           [s] cycle-level))
+
+(defn make-precond-hla [hierarchy var dst-val]
+  (SCH-Precond-HLA hierarchy [:!PRECOND var dst-val] var dst-val
+                   (--> hierarchy :dtg-to var dst-val)
+                   (--> hierarchy :ancestor-vars-set var)
+                   (when (--> hierarchy :cycle-to var dst-val)
+                     (--> hierarchy :var-levels var))))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Action HLAs ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+;[sas-problem predecessor-var-set ancestor-var-set var-levels dtg-to cycle-to greedy-optimization?]
+
+(deftype SDH-Action-HLA [hierarchy name action precond-hlas precond-var-set]
+  SDH-HLA
+  env/Action
+    (action-name     [] name)
+    (primitive?      [] false)
+  env/ContextualAction
+    (precondition-context [s] precond-var-set)
+  hierarchy/HighLevelAction
+    (immediate-refinements- [s]
+      (if (<= (count precond-hlas) 1)
+          [(concat precond-hlas [action])]
+        ***))
+    (cycle-level-           [s] nil))
+
+(defn make-action-hla [hierarchy action]
+  (let [effect-var (key (util/safe-singleton (:effect-map action)))]
+    (SDH-Action-HLA hierarchy [:!A (env/action-name action)] action
+                    (for [[var val] (:precond-map action)
+                          :when (not (= var effect-var))]
+                      (make-precond-hla hierarchy var val))
+                    (--> hierarchy :ancestor-vars-set effect-var))))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;; Precondition Set Nodes ;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+ ; This assumes no value-added needed:
+
+;; Idea: graph where a, b connected if precond sets intersect.
+ ; Arc is a->b if a superset of b, bid. if no containment.
+ ; any Topological order gives proper structure? (SCC-graph must be a tree!)
+; This loses on things like a -> b -> c , d -> c where we should get orering of a/b.
+
+;; TODO: recursively do these sets, somehow.
+
+
+(defn compute-precond-var-sets [init-sets par-effect-sets ordered-preconds]
+  (loop [init-sets init-sets, ordered-preconds ordered-preconds, ret []]
+    (if-let [[f & r] (seq ordered-preconds)]
+        (do (extend-hla! f init-sets par-effect-sets)
+            (recur (apply-effect-set-map init-sets (effect-sets f))
+                   r
+                   (conj ret [f (precond-var-set f)])))
+      ret)))
+
+(defn compute-interference-graph [preconds-and-var-sets]
+  (concat
+   (for [g preconds-and-var-sets] [::ROOT g])
+   (for [[p1 s1 :as g1] preconds-and-var-sets, [p2 s2 :as g2] preconds-and-var-sets
+         :when (and (not (empty? (clojure.set/intersection s1 s2)))
+                    (not (util/superset? s2 s1)))]
+     [g1 g2])))
+
+(defn compute-topo-map [init-sets par-effect-sets ordered-preconds]
+  (graphs/topological-sort-indices
+   (compute-interference-graph
+    (compute-precond-var-sets init-sets par-effect-sets ordered-preconds))))
+
+;; TODO: To properly know which vars to fiddle, must be able to separate direct effects from transitive ones.
+
+;; TODO: also must remember prec/effect dichotomy.
+
+(declare extend-precond-set-hla! make-interleaving-hla)
+;; Preconds are ordered, max-level (deepest) first.
+(deftype SDH-Precond-Set-HLA [precond-map ordered-preconds ref-atom] :as this
+  SDH-HLA
+    (precond-var-set [] (apply clojure.set/union        (map precond-var-set ordered-preconds)))
+    (effect-sets     [] (apply sequence-effect-set-maps (map effect-sets ordered-preconds)))
+    (pre-ref-pairs   [] [[{} @ref-atom]])
+    (compile-hla     [retain-type-set] (default-compile-hla this retain-type-set))
+    (hla-type        [] (type this))
+    (extend-hla!     [init-sets par-effect-sets]
+      (extend-precond-set-hla! this init-sets par-effect-sets))
+  env/Action
+    (action-name [] [:ps (System/identityHashCode this)])
+    (primitive?  [] false)
+  env/ContextualAction
+    (precondition-context [s] (precond-var-set this))
+  hierarchy/HighLevelAction
+    (immediate-refinements- [s] [@ref-atom])
+    (cycle-level-           [s] nil))
+
+;; For now, punt in several ways, only look for independent chunks, ...
+; Broken out so it can access bound vars.
+(defn extend-precond-set-hla! [hla init-sets par-effect-sets]
+  (let [{:keys [ordered-preconds ref-atom]} hla
+        topo-map (compute-topo-map init-sets par-effect-sets ordered-preconds)
+        chunks   (map #(map key %) (vals (util/group-by val topo-map)))]
+    (assert (= (first chunks) [::ROOT]))
+;    (println (map #(map (comp env/action-name first) %) (next chunks)))
+    (reset! ref-atom
+      (doall
+       (for [chunk (next chunks)]
+         (if (util/singleton? chunk)
+           (ffirst chunk)
+           (let [all-vars (apply concat (map second chunk))
+                 bad-vars (util/keyset (util/filter-map #(> (val %) 1) (util/frequencies all-vars)))
+                 par-effect-sets (select-keys *var-val-sets* bad-vars)]
+             (println "Interleaving HLAs with common ground:" bad-vars par-effect-sets)
+             (doseq [[hla] chunk] (extend-hla! hla {} par-effect-sets))
+             (assert (= (set all-vars) (set (apply concat (map second chunk)))))
+             (make-interleaving-hla (map (comp vector first) chunk) bad-vars))))))))
+
+(defn make-precond-set-hla [precond-map]
+  (SDH-Precond-Set-HLA
+   precond-map
+   (doall (map (partial apply make-precond-hla) (sort-by (comp - *var-levels* key) precond-map)))
+   (atom :dummy)))
+
+(defn trivially-satisfied-precond-set? [a s]
+  (if (= (type a) ::SDH-Compiled-HLA) (recur (:orig-hla a) s)
+      (and (= (type a) ::SDH-Precond-Set-HLA)
+           (every? (fn [p] (= (env/get-var s (:var p)) (:dst-val p)))
+                   (:ordered-preconds a)))))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Interleaving HLA ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+(declare make-interleaving-hla-refinement)
+
+(deftype SDH-Interleaving-HLA [refinements shared-var-set] :as this
+  SDH-HLA
+    (precond-var-set [] (apply clojure.set/union (map precond-var-set (apply concat refinements))))
+    (effect-sets     [] (throw (UnsupportedOperationException.)))
+    (pre-ref-pairs   [] (throw (UnsupportedOperationException.)))
+    (compile-hla     [retain-type-set]
+;      (println "interleaved compile" (map #(map env/action-name %) refinements))
+      (SDH-Interleaving-HLA. (vec (map #(compile-refinement % (conj retain-type-set ::SDH-Precond-Set-HLA)) refinements)) shared-var-set))
+    (hla-type        [] (type this))
+    (extend-hla!     [init-sets par-effect-sets] (throw (UnsupportedOperationException.)))
+  env/Action
+    (action-name     [] [:!I (map #(map env/action-name %) refinements) shared-var-set])
+    (primitive?      [] false)
+  env/ContextualAction
+    (precondition-context [s] (precond-var-set this))
+  hierarchy/HighLevelAction
+    (immediate-refinements- [s]
+      (let [[stalled-refs rest-refs] (split-with #(= (hla-type (first %)) ::SDH-Precond-Set-HLA) refinements)]
+;        (println "\n\n" s "\n" (map (comp env/action-name first) refinements))
+;        (println (map count [stalled-refs rest-refs]))
+;        (println (map #(map type %) rest-refs))
+        (if-let [[target-ref & more-refs] (seq rest-refs)]
+            (let [[first-a & rest-a] target-ref]
+              (for [ref (hierarchy/immediate-refinements first-a s)]
+                (make-interleaving-hla-refinement nil
+                  (concat stalled-refs [(concat ref rest-a)] more-refs)
+                  shared-var-set)))
+           ;; Interesting stuff here ... all refinements are blocked at precond set HLAs.
+           ;; First placeholder (not complete or smart): just pick one to do completely, no interleaving.
+          (if (and *greedy-optimization?* (some #(trivially-satisfied-precond-set? (first %) s) refinements))
+              (do ;(println "GREEDY") ;; TODO: is this the best place?
+                [(make-interleaving-hla-refinement
+                  [] ; (for [ref refinements :let [f (first ref)] :when (trivially-satisfied-precond-set? f s)] f)
+                  (for [ref refinements]
+                    (if (trivially-satisfied-precond-set? (first ref) s) (next ref) ref))
+                  shared-var-set)])
+            (for [i (range (count refinements))
+                  :let [[f & r] (nth refinements i)]]
+              (make-interleaving-hla-refinement [f]
+                (concat (subvec refinements 0 i) [r] (subvec refinements (inc i)))
+                shared-var-set))))))
+    (cycle-level-           [s] nil))
+
+(defn make-interleaving-hla [refinements shared-var-set]
+  (SDH-Interleaving-HLA (vec refinements) shared-var-set))
+
+;; Greedily pull irrelevant actions out, until we get to a normalized HLA
+(defn make-interleaving-hla-refinement [pre-actions refinements shared-var-set]
+  (loop [in-refinements refinements, out-refinements [], pre-actions pre-actions]
+    (if-let [[f & r] (seq in-refinements)]
+      (let [[front back] (split-with #(or (empty? (clojure.set/intersection shared-var-set (precond-var-set %)))
+                                          (env/primitive? %)) f)]
+        (recur r (if (seq back) (conj out-refinements back) out-refinements) (concat pre-actions front)))
+      (concat pre-actions (when (seq out-refinements) [(make-interleaving-hla out-refinements shared-var-set)])))))
+
+
+
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Top Level  ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+
+
+
+
+


Cleanup

Cleanup

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/ducky427/movies-cljs-rn.git/'


Add swap-pair util function

Add swap-pair util function

error: pathspec 'src/onyx/static/swap_pair.clj' did not match any file(s) known to git

--- src/onyx/static/swap_pair.clj
+++ src/onyx/static/swap_pair.clj
@@ -0,0 +1,16 @@
+(ns onyx.static.swap-pair)
+
+;; Adapted from Prismatic Plumbing:
+;; https://github.com/Prismatic/plumbing/blob/c53ba5d0adf92ec1e25c9ab3b545434f47bc4156/src/plumbing/core.cljx#L346-L361
+(defn swap-pair!
+  "Like swap! but returns a pair [old-val new-val]"
+  ([a f]
+     (loop []
+       (let [old-val @a
+             new-val (f old-val)]
+         (if (compare-and-set! a old-val new-val)
+           [old-val new-val]
+           (recur)))))
+  ([a f & args]
+     (swap-pair! a #(apply f % args))))
+


test: add send newsletter test

test: add send newsletter test

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/rodrigo-garcia-leon/semanario-universidad-newsletter.git/'


now using async success-messages for manual trigger. work remains to immediately pass on the rest of the result to UI so we can actually display a trigger button there

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/tobiasbayer/lambdacd.git/'


Typos, assemble!

Typos, assemble!

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/screen6/bigquery-clj.git/'


add offset validation to log impl

add offset validation to log impl

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/JetBrains/imperative-nodejs.git/'


Make it look pretty

Make it look pretty

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/starcity-properties/odin.git/'


[TH] denote missing field was optional

[TH] denote missing field was optional

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/voter-registration-works.git/'


Remove comment explaining previous JSON encoding workaround

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/turbovote-http-api.git/'


Initial checkin of spiral challenge stubs.

error: pathspec 'src/main/cljc/clojure_club/spiral_challenge/mbastian.cljc' did not match any file(s) known to git

--- src/main/cljc/clojure_club/spiral_challenge/mbastian.cljc
+++ src/main/cljc/clojure_club/spiral_challenge/mbastian.cljc
@@ -0,0 +1,4 @@
+(ns clojure-club.spiral-challenge.mbastian)
+
+
+(def seed [0])


Added first test.

Added first test.

error: pathspec 'project/scrape-summitpost-data/test/scrape_summitpost_data/search_result_test.clj' did not match any file(s) known to git

--- project/scrape-summitpost-data/test/scrape_summitpost_data/search_result_test.clj
+++ project/scrape-summitpost-data/test/scrape_summitpost_data/search_result_test.clj
@@ -0,0 +1,16 @@
+(ns scrape-summitpost-data.search-result-test
+  (require [clojure.test :refer [deftest is]]
+           [scrape-summitpost-data.search-result :as test-ns]))
+
+(deftest extract-last-page-test
+  (is (= 1
+         (#'test-ns/extract-last-page ["http://example.org?page=1"]))
+      "extracts value of page param")
+  (is (= 3
+         (#'test-ns/extract-last-page ["http://example.org?page=1"
+                                       "http://example.org?page=2"
+                                       "http://example.org?page=3"]))
+      "extracts highest value of page param")
+  (is (thrown? java.lang.NumberFormatException
+         (#'test-ns/extract-last-page ["http://example.org?page=f"]))
+      "fails on a non-integer value of page param"))


[WM] add replace template helper

[WM] add replace template helper

I'm using this to replace "\n" with "<br/>" in voting ID instructions.

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/notification-works.git/'


Restored mistakenly deleted Number IO problem

error: pathspec 'src/clojush/problems/software/number_io.clj' did not match any file(s) known to git

--- src/clojush/problems/software/number_io.clj
+++ src/clojush/problems/software/number_io.clj
@@ -0,0 +1,153 @@
+;; number_io.clj
+;; Tom Helmuth, thelmuth@cs.umass.edu
+;;
+;; Problem Source: iJava (http://ijava.cs.umass.edu/)
+;;
+;; This problem file defines the following problem:
+;; There are two inputs, a float and an int. The program must read them in,
+;; find their sum as a float, and print the result as a float.
+;;
+;; NOTE: input stack: in1 (float), in2 (int)
+;;       output stack: printed output
+
+(ns clojush.problems.software.number-io
+  (:use clojush.pushgp.pushgp
+        [clojush pushstate interpreter random util globals]
+        clojush.instructions.tag
+        clojure.math.numeric-tower
+        ))
+
+; Atom generators
+(def num-io-atom-generators
+  (concat (list
+            (fn [] (- (lrand-int 201) 100))
+            (fn [] (- (* (lrand) 200) 100.0))
+            (tag-instruction-erc [:float :integer] 1000)
+            (tagged-instruction-erc 1000)
+            ;;; end ERCs
+            'in1
+            'in2
+            ;;; end input instructions
+            )
+          (registered-for-stacks [:float :integer :print])))
+
+;; A list of data domains for the number IO problem. Each domain is a vector containing
+;; a "set" of inputs and two integers representing how many cases from the set
+;; should be used as training and testing cases respectively. Each "set" of
+;; inputs is either a list or a function that, when called, will create a
+;; random element of the set.
+(def num-io-data-domains
+  [[(fn [] (vector (- (* (lrand) 200) 100.0) (- (lrand-int 201) 100))) 25 1000] ;; Each input is a float and an int, both from range [-100,100]
+   ])
+
+;;Can make number-IO test data like this:
+;(test-and-train-data-from-domains num-io-data-domains)
+
+; Helper function for error function
+(defn num-io-test-cases
+  "Takes a sequence of inputs and gives IO test cases of the form
+   [[float-input int-input] output]."
+  [inputs]
+  (map #(vector %
+                (apply + %))
+       inputs))
+
+; Define error function. For now, each run uses different random inputs
+(defn num-io-error-function
+  "Returns the error function for the number IO problem. Takes as
+   input number IO data domains."
+  [data-domains]
+  (let [[train-cases test-cases] (map num-io-test-cases
+                                      (test-and-train-data-from-domains data-domains))]
+    (when true ;; Change to false to not print test cases
+      (doseq [[i case] (map vector (range) train-cases)]
+        (println (format "Train Case: %3d | Input/Output: %s" i (str case))))
+      (doseq [[i case] (map vector (range) test-cases)]
+        (println (format "Test Case: %3d | Input/Output: %s" i (str case)))))
+    (fn the-actual-num-io-error-function
+      ([program]
+        (the-actual-num-io-error-function program :train))
+      ([program data-cases] ;; data-cases should be :train or :test
+        (the-actual-num-io-error-function program data-cases false))
+      ([program data-cases print-outputs]
+        (let [behavior (atom '())
+              errors (flatten
+                       (doall
+                         (for [[[in-float in-int] out-float] (case data-cases
+                                                                          :train train-cases
+                                                                          :test test-cases
+                                                                          [])]
+                           (let [final-state (run-push program
+                                                       (->> (make-push-state)
+                                                         (push-item in-int :input)
+                                                         (push-item in-float :input)
+                                                         (push-item "" :output)))
+                                 printed-result (stack-ref :output 0 final-state)]
+                             (when print-outputs
+                               (println (format "Correct output: %-14s | Program output: %-14s" (pr-str (round-to-n-decimal-places out-float 10)) printed-result)))
+                             ; Record the behavior
+                             (when @global-print-behavioral-diversity
+                               (swap! behavior conj printed-result))
+                             ; Each test case results in two error values:
+                             ;   1. Numeric difference between correct output and the printed
+                             ;      output read into a float, rounded to 4 decimal places;
+                             ;      if such a conversion fails, the error is a penalty of 1000.
+                             ;   2. Levenstein distance between printed output and correct output as strings
+                             (vector (round-to-n-decimal-places
+                                       (try (min 1000.0 (abs (- out-float (Double/parseDouble printed-result))))
+                                         (catch Exception e 1000.0))
+                                       4)
+                                     (levenshtein-distance printed-result (pr-str (round-to-n-decimal-places out-float 10))))))))]
+          (when @global-print-behavioral-diversity
+            (swap! population-behaviors conj @behavior))
+          errors)))))
+
+(defn num-io-report
+  "Custom generational report."
+  [best population generation error-function report-simplifications]
+  (let [best-program (not-lazy (:program best))
+        best-test-errors (error-function best-program :test)
+        best-total-test-error (apply +' best-test-errors)]
+    (println ";;******************************")
+    (printf ";; -*- Number IO problem report - generation %s\n" generation)(flush)
+    (println "Test total error for best:" best-total-test-error)
+    (println (format "Test mean error for best: %.5f" (double (/ best-total-test-error (count best-test-errors)))))
+    (when (<= (:total-error best) 1.0E-4)
+      (doseq [[i [num-error lev-dist]] (map vector
+                                            (range)
+                                            (partition 2 best-test-errors))]
+        (println (format "Test Case  %3d | Numeric Error: %19.14f | Levenshtein Distance: %d" i (float num-error) lev-dist))))
+    (println ";;------------------------------")
+    (println "Outputs of best individual on training cases:")
+    (error-function best-program :train true)
+    (println ";;******************************")
+    )) ;; To do validation, could have this function return an altered best individual
+       ;; with total-error > 0 if it had error of zero on train but not on validation
+       ;; set. Would need a third category of data cases, or a defined split of training cases.
+
+
+; Define the argmap
+(def argmap
+  {:error-function (num-io-error-function num-io-data-domains)
+   :atom-generators num-io-atom-generators
+   :max-points 200
+   :max-points-in-initial-program 100
+   :evalpush-limit 200
+   :population-size 1000
+   :max-generations 200
+   :parent-selection :lexicase
+   :epigenetic-markers []
+   :genetic-operator-probabilities {:alternation 0.3
+                                    :uniform-mutation 0.2
+                                    [:alternation :uniform-mutation] 0.5
+                                    }
+   :alternation-rate 0.01
+   :alignment-deviation 5
+   :uniform-mutation-rate 0.01
+   :problem-specific-report num-io-report
+   :print-behavioral-diversity true
+   :report-simplifications 0
+   :final-report-simplifications 5000
+   :max-error 5000
+   :error-threshold 1.0E-4
+   })


[WM] test all the things

[WM] test all the things

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/democracyworks/election-notification-http-api.git/'


Copy in elections API tests

Copy in elections API tests


Pass -Dstorm.log.dir arg to LogWriter

Pass -Dstorm.log.dir arg to LogWriter

--- storm-core/src/clj/backtype/storm/daemon/supervisor.clj
+++ storm-core/src/clj/backtype/storm/daemon/supervisor.clj
@@ -696,6 +696,7 @@
                      (str "-Dstorm.id=" storm-id)
                      (str "-Dworker.id=" worker-id)
                      (str "-Dworker.port=" port)
+                     (str "-Dstorm.log.dir=" storm-log-dir)
                      (str "-Dlog4j.configurationFile=" storm-log4j2-conf-dir file-path-separator "worker.xml")
                      "backtype.storm.LogWriter"]
                     [(java-cmd) "-server"]


facebook captcha to run from cmd

facebook captcha to run from cmd

error: pathspec 'examples/src/queries/captcha_facebook.clj' did not match any file(s) known to git

--- examples/src/queries/captcha_facebook.clj
+++ examples/src/queries/captcha_facebook.clj
@@ -0,0 +1,31 @@
+(ns queries.captcha-facebook
+  (:require [anglican.runtime :refer :all]
+            [anglican.emit :refer [defquery with-primitive-procedures]]
+            [helpers.captcha-facebook :refer [render abc-dist abc-sigma letter-dict]]))
+
+;; QUERY
+(with-primitive-procedures [render abc-dist repeatedly]
+  (defquery captcha-facebook [baseline-image]
+    (let [;; Number of letters in CAPTCHA
+          num-letters (sample "numletters" (uniform-discrete 6 8))
+          font-size (sample "fontsize" (uniform-discrete 38 44))
+          kerning (sample "kerning" (uniform-discrete -2 2))
+          letter-ids (repeatedly num-letters #(sample "letterid" (uniform-discrete 0 (count letter-dict))))
+          letters (apply str (map (partial nth letter-dict) letter-ids))
+
+          ;; Render image using renderer from ...
+          rendered-image (render letters font-size kerning)]
+
+      ;; ABC observe
+      (observe (abc-dist rendered-image abc-sigma) baseline-image)
+
+      ;; Returns
+      {:letters letters
+       :font-size font-size
+       :kerning kerning})))
+
+;; COMPILE stuff
+(defn COMPILE-combine-observes-fn [observes]
+  (:value (first observes)))
+
+(def COMPILE-query-args [nil])


improve icon organization

improve icon organization

remote: Support for password authentication was removed on August 13, 2021.
remote: Please see https://docs.github.com/en/get-started/getting-started-with-git/about-remote-repositories#cloning-with-https-urls for information on currently recommended modes of authentication.
fatal: Authentication failed for 'https://github.com/wilkerlucio/youtube-looper.git/'