From e263ed10c64c1e124e7c0b12e686d178920fa712 Mon Sep 17 00:00:00 2001 From: Dave Della Costa Date: Tue, 10 Jun 2014 12:12:21 +0900 Subject: [PATCH] better testing documentation; adds basic subscription functionality back to base-subscribed-views; beginning of bulk update functionality; moved all database table creation into SQL script --- README.md | 15 +++- project.clj | 4 +- src/views/base_subscribed_views.clj | 91 +++++++++++++++++------ src/views/db/core.clj | 3 +- src/views/db/load.clj | 2 +- src/views/filters.clj | 34 --------- src/views/subscribed_views.clj | 1 - src/views/subscriptions.clj | 15 +++- test/views/all_tests.clj | 2 + test/views/base_subscribed_views_test.clj | 45 +++++++++++ test/views/db/core_test.clj | 28 ++++--- test/views/db/load_test.clj | 2 +- test/views/fixtures.clj | 33 ++------ test/views/subscriptions_test.clj | 10 +++ test/views/test_db.sql | 12 +++ 15 files changed, 198 insertions(+), 99 deletions(-) delete mode 100644 src/views/filters.clj create mode 100644 test/views/base_subscribed_views_test.clj create mode 100644 test/views/test_db.sql diff --git a/README.md b/README.md index 7700b58..b098ebb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,20 @@ TODO ## Testing -You can run all tests in the repl with +You will need to set up the test db to run the tests: + +```bash +$ psql -Upostgres < test/views/test_db.sql +CREATE ROLE +CREATE DATABASE +$ +``` + +This will create a role `views_user` and a database owned by that user called `views_test`. + +(You can change the database settings if you'd like by editing that file and checking the config in `test/views/fixtures.clj`.) + +Then, to run all tests in the repl: ```clojure user=> (require '[views.all-tests :as at]) diff --git a/project.clj b/project.clj index 0d9bd97..083ca45 100644 --- a/project.clj +++ b/project.clj @@ -11,7 +11,9 @@ [org.clojure/core.async "0.1.303.0-886421-alpha"] [org.clojure/java.jdbc "0.3.3"] [honeysql "0.4.3"] - [org.postgresql/postgresql "9.2-1003-jdbc4"]] + [edl "0.1.0"] + [org.postgresql/postgresql "9.2-1003-jdbc4"] + [clj-logging-config "1.9.10"]] :profiles {:test {:dependencies [[org.clojure/tools.nrepl "0.2.3"] [environ "0.4.0"] diff --git a/src/views/base_subscribed_views.clj b/src/views/base_subscribed_views.clj index 9720f65..c2c9585 100644 --- a/src/views/base_subscribed_views.clj +++ b/src/views/base_subscribed_views.clj @@ -1,36 +1,85 @@ (ns views.base-subscribed-views (:require - [views.db.core :refer [initial-views]] - [views.subscribed-views :refer [SubscribedViews subscriber-key-fn prefix-fn send-message]] - [views.subscriptions :as vs :refer [add-subscriptions!]] + [views.db.load :refer [initial-views]] + [views.subscribed-views :refer [SubscribedViews subscriber-key-fn prefix-fn]] + [views.subscriptions :as vs :refer [add-subscriptions! remove-subscription! subscriptions-for]] [clojure.tools.logging :refer [debug info warn error]] [clojure.core.async :refer [put! opts first :unsafe) + (do (warn "YOU ARE RUNNING IN UNSAFE MODE, AND NO FILTERS ARE PRESENT FOR VIEW-SIG: " %) + true) + (throw (Exception. (str "No filter set for view " %)))))) + view-sigs))) + +(defn send-message + [this address msg] + (warn "IMPLEMENT ME. Got message " msg " sent to address " address)) + +(deftype BaseSubscribedViews [db templates send-fn broadcast-fn subscribed-views-fn opts] SubscribedViews (subscribe-views [this sub-req] - ;; (let [view-sigs (view-filter sub-req (:body sub-req) templates)] ; this is where security comes in. (let [subscriber-key (subscriber-key-fn this sub-req) - view-sigs (:view-sigs sub-req)] - (info "Subscribing views: " view-sigs) + view-sigs (view-filter sub-req (:views sub-req) templates opts)] ; this is where security comes in. + (info "Subscribing views: " view-sigs " for subscriber " subscriber-key) (when (seq view-sigs) - (add-subscriptions! subscriber-key view-sigs (prefix-fn this sub-req)) - (thread - (->> (initial-views db view-sigs templates @vs/compiled-views) - (send-message this subscriber-key)))))) + (let [subbed-views (if-let [prefix (prefix-fn this sub-req)] + (add-subscriptions! subscriber-key view-sigs templates prefix) + (add-subscriptions! subscriber-key view-sigs templates))] + (thread + (->> (initial-views db view-sigs templates subbed-views) + ((if send-fn send-fn send-message) this subscriber-key))))))) - (unsubscribe-views [this unsub-req]) + (unsubscribe-views + [this unsub-req] + (let [subscriber-key (subscriber-key-fn this unsub-req) + view-sigs (:views unsub-req)] + (info "Unsubscribing views: " view-sigs " for subscriber " subscriber-key) + (if-let [prefix (prefix-fn this unsub-req)] + (doseq [vs view-sigs] (remove-subscription! subscriber-key vs prefix)) + (doseq [vs view-sigs] (remove-subscription! subscriber-key vs))))) - (disconnect [this disconnect-req]) - - (subscribed-views [this] @vs/compiled-views) - - (broadcast-deltas [this fdb views-with-deltas]) - - (send-message [this address msg] - (warn "IMPLEMENT ME. Got message " msg " sent to address " address)) + (disconnect [this disconnect-req] + (let [subscriber-key (:subscriber-key disconnect-req) + prefix (prefix-fn this disconnect-req) + view-sigs (if prefix (subscriptions-for subscriber-key prefix) (subscriptions-for subscriber-key))] + (if prefix + (doseq [vs view-sigs] (remove-subscription! subscriber-key vs prefix)) + (doseq [vs view-sigs] (remove-subscription! subscriber-key vs))))) (subscriber-key-fn [this msg] (:subscriber-key msg)) - (prefix-fn [this msg] nil)) + (prefix-fn [this msg] nil) + + ;; DB interaction + (subscribed-views [this] @vs/compiled-views) + + (broadcast-deltas [this fdb views-with-deltas])) diff --git a/src/views/db/core.clj b/src/views/db/core.clj index d95203b..dc94e6d 100644 --- a/src/views/db/core.clj +++ b/src/views/db/core.clj @@ -72,6 +72,7 @@ (merge {:args (rest view-sig) :view-sig view-sig :view compiled-view + :bulk-update? (:bulk-update? (meta view-template)) :tables (set (vh/extract-tables compiled-view))} (compile-dummy-view view-template (rest view-sig))))) @@ -444,7 +445,7 @@ (broadcast-deltas ~subscribed-views ~(second binding) @deltas#) result#)))))) -(defn vaction! +(defn vexec! "Used to perform arbitrary insert/update/delete actions on the database, while ensuring that view deltas are appropriately checked and calculated for the currently registered views as reported by a type implementing diff --git a/src/views/db/load.clj b/src/views/db/load.clj index 86b29da..eb4100a 100644 --- a/src/views/db/load.clj +++ b/src/views/db/load.clj @@ -23,7 +23,7 @@ (reduce (fn [results nv] (->> (get subscribed-views nv) - :view-map + :view (view-query db) (into []) (post-process-result-set nv templates) diff --git a/src/views/filters.clj b/src/views/filters.clj deleted file mode 100644 index a63c67a..0000000 --- a/src/views/filters.clj +++ /dev/null @@ -1,34 +0,0 @@ -(ns views.filters) - -(defn view-filter - "Takes a subscription request msg, a collection of view-sigs and - the config templates hash-map for an app. Checks if there is - a global filter-fn in the hash-map metadata and checks against - that if it exists, as well as against any existing filter - functions for individual template config entries. Template - config hash-map entries can specify a filter-fn using the key - :filter-fn, and the global filter-fn is the same, only on - the config meta-data (i.e. (with-meta templates {:filter-fn ...})) - - By default throws an exception if no filters are present. - By passing in {:unsafe true} in opts, this can be overridden." - [msg view-sigs templates & opts] - (let [global-filter-fn (:filter-fn (meta templates))] - (filterv - #(let [filter-fn (:filter-fn (get templates (first %)))] - (cond - (and filter-fn global-filter-fn) - (and (global-filter-fn msg %) (filter-fn msg %)) - - filter-fn - (filter-fn msg %) - - global-filter-fn - (global-filter-fn msg %) - - :else - (if (-> opts first :unsafe) - (do (warn "YOU ARE RUNNING IN UNSAFE MODE, AND NO FILTERS ARE PRESENT FOR VIEW-SIG: " %) - true) - (throw (Exception. (str "No filter set for view " %)))))) - view-sigs))) diff --git a/src/views/subscribed_views.clj b/src/views/subscribed_views.clj index 63cc1ac..6457c4a 100644 --- a/src/views/subscribed_views.clj +++ b/src/views/subscribed_views.clj @@ -5,7 +5,6 @@ (subscribe-views [this sub-request]) (unsubscribe-views [this unsub-request]) (disconnect [this disconnect-request]) - (send-message [this address msg]) (subscriber-key-fn [this msg]) (prefix-fn [this msg]) diff --git a/src/views/subscriptions.clj b/src/views/subscriptions.clj index d821e9f..2528cfd 100644 --- a/src/views/subscriptions.clj +++ b/src/views/subscriptions.clj @@ -36,10 +36,17 @@ ([subscriber-key view-sigs templates] (add-subscriptions! subscriber-key view-sigs templates nil)) ([subscriber-key view-sigs templates prefix] - (doseq [vs view-sigs] - (if prefix - (add-subscription! subscriber-key vs templates prefix) - (add-subscription! subscriber-key vs templates))))) + (last (mapv + #(if prefix + (add-subscription! subscriber-key % templates prefix) + (add-subscription! subscriber-key % templates)) + view-sigs)))) + +(defn subscriptions-for + ([subscriber-key] + (reduce #(if (contains? (second %2) subscriber-key) (conj %1 (first %2)) %1) [] @subscribed-views)) + ([subscriber-key prefix] + (reduce #(if (contains? (second %2) subscriber-key) (conj %1 (first %2)) %1) [] (get @subscribed-views prefix)))) (defn subscribed-to ([view-sig] diff --git a/test/views/all_tests.clj b/test/views/all_tests.clj index 6ebef9a..bfcb031 100644 --- a/test/views/all_tests.clj +++ b/test/views/all_tests.clj @@ -2,6 +2,7 @@ (:require [clojure.test :refer [run-tests]] [views.subscriptions-test] + [views.base-subscribed-views-test] [views.db.core-test] [views.db.honeysql-test] [views.db.load-test])) @@ -9,6 +10,7 @@ (defn run-all-tests [] (run-tests 'views.subscriptions-test + 'views.base-subscribed-views-test 'views.db.core-test 'views.db.honeysql-test 'views.db.load-test)) diff --git a/test/views/base_subscribed_views_test.clj b/test/views/base_subscribed_views_test.clj new file mode 100644 index 0000000..9be8331 --- /dev/null +++ b/test/views/base_subscribed_views_test.clj @@ -0,0 +1,45 @@ +(ns views.base-subscribed-views-test + (:require + [views.base-subscribed-views :as bsv] ; :refer [BaseSubscribedViews]] + [views.subscribed-views :refer [SubscribedViews subscriber-key-fn prefix-fn subscribe-views unsubscribe-views disconnect]] + [views.subscriptions :as vs :refer [subscribed-to?]] + [views.fixtures :as vf] + [clojure.test :refer [use-fixtures deftest is]] + [clojure.java.jdbc :as j] + [clj-logging-config.log4j :refer [set-logger! set-loggers!]]) + (:import + [views.base_subscribed_views BaseSubscribedViews])) + +(set-loggers! "views.base-subscribed-views" {:level :error}) + +(defn- subscription-fixtures! + [f] + (reset! vs/subscribed-views {}) + (reset! vs/compiled-views {}) + (f)) + +(use-fixtures :each vf/database-fixtures! subscription-fixtures!) + +(deftest subscribes-and-dispatches-initial-view-result-set + (let [send-fn #(is (and (= %2 1) (= %3 {[:users] []}))) + base-subbed-views (BaseSubscribedViews. vf/db vf/templates send-fn nil nil {:unsafe true})] + (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]}))) + +(deftest unsubscribes-view + (let [base-subbed-views (BaseSubscribedViews. vf/db vf/templates nil nil nil {:unsafe true})] + (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]}) + (unsubscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]}) + (is (not (subscribed-to? 1 [:users]))))) + +(deftest filters-subscription-requests + (let [templates (assoc-in vf/templates [:users :filter-fn] (fn [msg _] (:authorized? msg))) + base-subbed-views (BaseSubscribedViews. vf/db templates nil nil nil nil)] + (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]}) + (is (not (subscribed-to? 1 [:users]))))) + +(deftest removes-all-subscriptions-on-disconnect + (let [base-subbed-views (BaseSubscribedViews. vf/db vf/templates nil nil nil {:unsafe true})] + (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users][:user-posts 1]]}) + (disconnect base-subbed-views {:subscriber-key 1}) + (is (not (subscribed-to? 1 [:user-posts 1]))) + (is (not (subscribed-to? 1 [:users]))))) diff --git a/test/views/db/core_test.clj b/test/views/db/core_test.clj index 5db42bc..e84cdad 100644 --- a/test/views/db/core_test.clj +++ b/test/views/db/core_test.clj @@ -1,9 +1,14 @@ (ns views.db.core-test (:require [clojure.test :refer [deftest is run-tests]] + [edl.core :refer [defschema]] [honeysql.core :as hsql] [honeysql.helpers :as hh] - [views.db.core :as vdb])) + [views.fixtures :as vf] + [views.db.core :as vdb] + [views.base-subscribed-views :as bsv]) + (:import + [views.base_subscribed_views BaseSubscribedViews])) (defn join-test-template [id val3] @@ -53,14 +58,6 @@ (is (= (hsql/format check-template) ["SELECT f.id, f.val3 FROM foo f INNER JOIN bar b ON b.id = f.b_id LEFT JOIN baz ba ON ba.id = b.ba_id RIGHT JOIN qux q ON q.id = ba.q_id WHERE (b.id = 123 AND f.val2 = ?)" "constant"])))) -;; ;; Not meaningful at this point perhaps...view-check-template shouldn't -;; ;; get handed an action that doesn't have a related table in the first place...? -;; (deftest removes-non-related-tables -;; (let [update-bar (update-bar-template "foo" [:= :id 123]) -;; vm (vdb/view-map no-where-view-template [:no-where]) -;; check-template (:view-check (vdb/view-check-template vm update-bar))] -;; (is (nil? check-template)))) - (deftest creates-collection-of-views-to-check (let [views [(vdb/view-map no-where-view-template [:no-where]) ; no :bar (vdb/view-map no-where-view-template [:no-where]) ; no :bar @@ -75,4 +72,17 @@ ;; and 1 for *both* the joint-test-templates. (is (= (count checked-views) 2)))) +;; What is this for? (def left-join-example (hsql/build :select [:R.a :S.C] :from :R :left-join [:S [:= :R.B :S.B]] :where [:!= :S.C 20])) + +(deftest notes-view-map-as-no-delta-calc + (let [tmpl (with-meta vf/users-tmpl {:bulk-update? true})] + (is (:bulk-update? (vdb/view-map tmpl [:users]))))) + +(defschema schema vf/db "public") + +;; (deftest sends-entire-view-on-every-update-with-bulk-update +;; (let [tmpl (with-meta vf/users-tmpl {:bulk-update? true}) +;; vm (vdb/view-map tmpl [:users]) +;; bsv (BaseSubscribedViews. vf/db + diff --git a/test/views/db/load_test.clj b/test/views/db/load_test.clj index 8b3c089..b7ede2b 100644 --- a/test/views/db/load_test.clj +++ b/test/views/db/load_test.clj @@ -10,7 +10,7 @@ (defn subscribed-views [] - {[:users] {:view-map ((get-in templates [:users :fn]))}}) + {[:users] {:view ((get-in templates [:users :fn]))}}) (deftest initializes-views (let [users (gen-n-users! 2)] diff --git a/test/views/fixtures.clj b/test/views/fixtures.clj index 64d45fa..3661335 100644 --- a/test/views/fixtures.clj +++ b/test/views/fixtures.clj @@ -5,9 +5,6 @@ [honeysql.core :as hsql] [clojure.data.generators :as dg])) -;; CREATE ROLE views_user LOGIN PASSWORD 'password'; -;; CREATE DATABASE views_test OWNER views_user; - (defn sql-ts ([ts] (java.sql.Timestamp. ts)) ([] (java.sql.Timestamp. (.getTime (java.util.Date.))))) @@ -18,30 +15,15 @@ :user (get :views-test-user e/env "views_user") :password (get :views-test-ppassword e/env "password")}) -(defn users-table-fixture! - [] - (j/execute! db ["CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL, created_on DATE NOT NULL)"])) - -(defn posts-table-fixture! - [] - (j/execute! db ["CREATE TABLE posts (id SERIAL PRIMARY KEY, - title TEXT NOT NULL, - body TEXT NOT NULL, - created_on DATE NOT NULL, - user_id INTEGER NOT NULL, - FOREIGN KEY (user_id) REFERENCES users(id))"])) - -(defn drop-tables! +(defn clean-tables! [tables] - (doseq [t tables] - (j/execute! db [(str "DROP TABLE " (name t))]))) + (doseq [t (map name tables)] + (j/execute! db [(str "DELETE FROM " t)]))) (defn database-fixtures! [f] - (users-table-fixture!) - (posts-table-fixture!) - (f) - (drop-tables! [:posts :users])) + (clean-tables! [:posts :users]) + (f)) (defn user-fixture! [name] @@ -59,9 +41,10 @@ (defn user-posts-tmpl [user_id] - (hsql/build :select [:u.user_id :u.name :p.title :p.body :p.created_on] + (hsql/build :select [:u.id :u.name :p.title :p.body :p.created_on] :from {:posts :p} - :join [[:users :u][:= :user_id user_id]])) + :join [[:users :u][:= :u.id :p.user_id]] + :where [:= :p.user_id user_id])) (def templates {:users {:fn #'users-tmpl} diff --git a/test/views/subscriptions_test.clj b/test/views/subscriptions_test.clj index b4b1db2..0a9645a 100644 --- a/test/views/subscriptions_test.clj +++ b/test/views/subscriptions_test.clj @@ -64,3 +64,13 @@ (vs/add-subscription! key view-sig templates) (vs/remove-subscription! key view-sig) (is (nil? (vs/compiled-view-for [:user-posts 1]))))) + +(deftest retrieves-subscriptions-for-subscriber + (let [key 1, view-sigs [[:users][:user-posts 1]]] + (vs/add-subscriptions! key view-sigs templates) + (is (= (set (vs/subscriptions-for 1)) (set view-sigs))))) + +(deftest retrieves-subscriptions-for-subscriber-with-prefix + (let [key 1, view-sigs [[:users][:user-posts 1]] prefix 1] + (vs/add-subscriptions! key view-sigs templates prefix) + (is (= (set (vs/subscriptions-for 1 prefix)) (set view-sigs))))) diff --git a/test/views/test_db.sql b/test/views/test_db.sql new file mode 100644 index 0000000..51f0bae --- /dev/null +++ b/test/views/test_db.sql @@ -0,0 +1,12 @@ +CREATE ROLE views_user LOGIN PASSWORD 'password'; +CREATE DATABASE views_test OWNER views_user; +\c postgresql://localhost/views_test; +CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL, created_on DATE NOT NULL); +CREATE TABLE posts (id SERIAL PRIMARY KEY, + title TEXT NOT NULL, + body TEXT NOT NULL, + created_on DATE NOT NULL, + user_id INTEGER NOT NULL, + FOREIGN KEY (user_id) REFERENCES users(id)); +ALTER TABLE users OWNER TO views_user; +ALTER TABLE posts OWNER TO views_user;