diff --git a/.gitignore b/.gitignore index 46d8b9a..1ec87ad 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ pom.xml.asc /.lein-* /.nrepl-port *~ +*.bk diff --git a/src/views/db/checks.clj b/src/views/db/checks.clj index d9264d8..c6f2902 100644 --- a/src/views/db/checks.clj +++ b/src/views/db/checks.clj @@ -1,5 +1,7 @@ (ns views.db.checks (:require + [views.db.honeysql :as vh] + [clojure.set :refer [intersection]] [clojure.zip :as z] [zip.visit :as zv] [honeysql.core :as hsql])) @@ -32,3 +34,13 @@ (-> q (update-in [:where] #(merge % (:where action))) (assoc :select (mapv second p))))) + +(defn have-overlapping-tables? + "Takes two Honeysql hash-maps, one for action, one for view, and returns + boolean value representing whether or not their set of tables intersect." + [action view] + (->> [action view] + (map (comp set #(map first %) vh/extract-tables)) + (apply intersection) + seq + boolean)) diff --git a/src/views/db/core.clj b/src/views/db/core.clj index 7c2b64b..5413065 100644 --- a/src/views/db/core.clj +++ b/src/views/db/core.clj @@ -4,133 +4,9 @@ (:require [clojure.java.jdbc :as j] [clojure.tools.logging :refer [debug]] - [views.db.load :as vdbl] - [views.db.honeysql :as vh] [views.db.deltas :as vd] [views.subscribed-views :refer [subscribed-views broadcast-deltas]])) -;; -;; Takes the HoneySQL template for a view and the arglist -;; and compiles the view with a set of dummy args in the -;; format -;; [?1, ?2, ?3 ... ?N] -;; -;; Returns a map of the compiled hash-map and the args -;; with keys :dummy-view and :dummy-args respectively. -;; -(defn- compile-dummy-view - [view-template args] - (let [dummy-args (take (count args) (range)) - dummy-args (map #(str "?" %) dummy-args)] - {:dummy-view (apply view-template dummy-args) - :dummy-args dummy-args})) - -;; -;; Terminology and data structures used throughout this code -;; -;; -template - refers to a function which receives parameters -;; and returns a HoneySQL hash-map with params interpolated. -;; -;; action - describes the HoneySQL hash-map for the action to be performed -;; --the template function has already been called and returned this -;; with the appropriate parameter arguments. -;; -;; view-map - contains a set of computed information for each view itself. -;; Refer to the view-map doc-string for more information. -;; -;; view-check - SQL for checking whether or not a view needs to receive deltas -;; upon completion of an operation. -;; - -(defn view-map - "Constructs a view map from a HoneySQL view function and its arguments. - Contains four fields: - :view - the hash-map with interpolated parameters - :view-sig - the \"signature\" for the view, i.e. [:matter 1] - :args - the arguments passed in. - :tables - the tables present in all :from, :insert-into, - :update, :delete-from, :join, :left-join :right-join clauses - - Input is a view template function and a view signature. The template - function must take the same number of paramters as the signature and - return a honeysql data structure " - [view-template view-sig] - (let [compiled-view (if (> (count view-sig) 1) - (apply view-template (rest view-sig)) - (view-template))] - (merge {:args (rest view-sig) - :view-sig view-sig - :view compiled-view - :refresh-only? (:refresh-only (meta view-template)) - :tables (set (vh/extract-tables compiled-view))} - (compile-dummy-view view-template (rest view-sig))))) - -(defn view-sig->view-map - "Takes a map of sig keys to view template function vars (templates) - and a view signature (view-sig the key for the template map and its args) - and returns a view-map for that view-sig." - [templates view-sig] - (let [lookup (first view-sig)] - (view-map (get-in templates [lookup :fn]) view-sig))) - -(defn update-deltas-with-refresh-set - [refresh-set] - (fn [view-deltas] - (if (coll? view-deltas) - (map #(assoc % :refresh-set refresh-set) view-deltas) - [{:refresh-set refresh-set}]))) - -(defn calculate-refresh-sets - "For refresh-only views, calculates the refresh-set and adds it to the view's delta update collection." - [deltas db templates refresh-only-views] - (reduce - (fn [d {:keys [view-sig view] :as rov}] - (let [refresh-set (get (vdbl/initial-view db view-sig templates view) view-sig)] - (update-in d [view-sig] (update-deltas-with-refresh-set refresh-set)))) - deltas - refresh-only-views)) - -(defn format-deltas - "Removes extraneous data from view delta response collections." - [views-with-deltas] - (->> views-with-deltas - (map #(select-keys % [:view-sig :delete-deltas :insert-deltas :refresh-set])) - (group-by :view-sig))) - -(defn do-view-transaction - "Takes the following arguments: - schema - from edl.core/defschema - db - clojure.java.jdbc database connection - all-views - the current set of views (view-maps--see view-map fn docstring for - description) in memory for the database - action - the HoneySQL pre-SQL hash-map with parameters already interpolated. - - The function will then perform the following sequence of actions, all run - within a transaction (with isolation serializable): - - 1) Create pre-check SQL for each view in the list. - 2) Run the pre-check SQL (or fail out based on some simple heuristics) to - identify if we want to send delta messages to the view's subscribers - (Note: this happens after the database action for *inserts only*). - 3) Run the database action (insert/action/delete). - 4) Calculate deltas based on the method described in section 5.4, \"Rule Generation\" - of the paper \"Deriving Production Rules for Incremental Rule Maintenance\" - by Stefano Ceri and Jennifer Widom (http://ilpubs.stanford.edu:8090/8/1/1991-4.pdf) - - The function returns the views which received delta updates with the deltas - keyed to each view-map at the keys :insert-deltas and :delete-deltas." - [schema db all-views action templates] - ;; Every update connected with a view is done in a transaction: - (j/with-db-transaction [t db :isolation :serializable] - (let [{full-refresh-views true normal-views nil} (group-by :refresh-only? all-views) - need-deltas (vd/do-view-pre-checks t normal-views action) - need-deltas (map #(vd/generate-view-delta-map % action) need-deltas) - table (-> action vh/extract-tables ffirst) - pkey (vd/get-primary-key schema table) - {:keys [views-with-deltas result-set]} (vd/perform-action-and-return-deltas schema t need-deltas action table pkey) - deltas (calculate-refresh-sets (format-deltas views-with-deltas) t templates full-refresh-views)] - {:new-deltas deltas :result-set result-set}))) - ;; ;; Need to catch this and retry: ;; java.sql.SQLException: ERROR: could not serialize access due to concurrent update @@ -198,7 +74,7 @@ - broadcast-deltas takes ... ." [{:keys [db schema base-subscribed-views templates namespace] :as conf} action-map] (let [subbed-views (subscribed-views base-subscribed-views namespace) - transaction-fn #(do-view-transaction schema db subbed-views action-map templates)] + transaction-fn #(vd/do-view-transaction schema db subbed-views action-map templates)] (if-let [deltas (:deltas db)] ;; inside a transaction we just collect deltas and do not retry (let [{:keys [new-deltas result-set]} (transaction-fn)] (swap! deltas into new-deltas) diff --git a/src/views/db/deltas.clj b/src/views/db/deltas.clj index 4650e0d..81446fa 100644 --- a/src/views/db/deltas.clj +++ b/src/views/db/deltas.clj @@ -4,8 +4,53 @@ [clojure.java.jdbc :as j] [honeysql.core :as hsql] [honeysql.helpers :as hh] + [views.db.load :as vdbl] + [views.db.checks :as vc] [views.db.honeysql :as vh])) +;; +;; Terminology and data structures used throughout this code +;; +;; -template - refers to a function which receives parameters +;; and returns a HoneySQL hash-map with params interpolated. +;; +;; action - describes the HoneySQL hash-map for the action to be performed +;; --the template function has already been called and returned this +;; with the appropriate parameter arguments. +;; +;; view-map - contains a set of computed information for each view itself. +;; Refer to the view-map doc-string for more information. +;; + +(defn view-map + "Constructs a view map from a HoneySQL view function and its arguments. + Contains four fields: + :view - the hash-map with interpolated parameters + :view-sig - the \"signature\" for the view, i.e. [:matter 1] + :tables - the tables present in all :from, :insert-into, + :update, :delete-from, :join, :left-join :right-join clauses + + Input is a view template function and a view signature. The template + function must take the same number of paramters as the signature and + return a honeysql data structure " + [view-template view-sig] + (let [compiled-view (if (> (count view-sig) 1) + (apply view-template (rest view-sig)) + (view-template))] + {:view-sig view-sig + :view compiled-view + :refresh-only? (:refresh-only (meta view-template))})) + +(defn view-sig->view-map + "Takes a map of sig keys to view template function vars (templates) + and a view signature (view-sig the key for the template map and its args) + and returns a view-map for that view-sig." + [templates view-sig] + (let [lookup (first view-sig)] + (view-map (get-in templates [lookup :fn]) view-sig))) + +;; Helpers + (defn get-primary-key "Get a primary key for a table." [schema table] @@ -13,49 +58,11 @@ (keyword (get-in schema [(name table) :primary-key :column_name])) (throw (Exception. (str "Cannot find primary key for table: " table))))) -(defn swap-out-dummy-for-pos - "Replaces dummy arg like \"?0\" for integer value (0) so we can sort args." - [dummy-arg] - (Integer. (subs dummy-arg 1))) - -;; Helper for determine-filter-clauses (which is a helper -;; for view-check-template). Extracts constituent parts from -;; where clause. -(defn set-filter-clauses - [dummy-args fc w] - (if (= w :and) - fc - (if (contains? (set dummy-args) (last w)) - (update-in fc [:s] assoc (swap-out-dummy-for-pos (last w)) (second w)) - (update-in fc [:w] (fnil conj []) w)))) - -;; Helper for generating the view-check HoneySQL template. -;; Builds the where and select clauses up from constituent -;; where-clauses. Placeholder identifies the parameters -;; to pull out into the select clause. -(defn determine-filter-clauses - [wc dummy-args] - (let [fc {:s {} :w nil} - fc (if (and (not= :and (first wc)) (not (coll? (first wc)))) - (set-filter-clauses dummy-args fc wc) - (reduce #(set-filter-clauses dummy-args %1 %2) fc wc))] - (-> fc - (update-in [:s] #(into [] (vals (sort-by key %)))) - (update-in [:w] #(vh/with-op :and %))))) - -(defn append-arg-map - "Removes table/alias namespacing from select fields and creates a hash-map - of field to arguments for checking this view against checked-results later on. - Note that this assumes our select-fields are in the same order as they - are present in the :args view-map field (which they should be)." - [view-map select-fields] - (let [select-fields (map #(-> % name (split #"\.") last keyword) select-fields)] - (assoc view-map :arg-compare (zipmap select-fields (into [] (:args view-map)))))) - (defn- create-view-delta-where-clauses [view-map action] - (let [action-table (first (vh/extract-tables action))] - (for [view-table (vh/find-table-aliases action-table (:tables view-map))] + (let [action-table (first (vh/extract-tables action)) + view-tables (vh/extract-tables (:view view-map))] + (for [view-table (vh/find-table-aliases action-table view-tables)] (-> (:where action) (vh/prefix-columns (vh/table-alias view-table)) (vh/replace-table (vh/table-alias action-table) (vh/table-alias view-table)))))) @@ -71,108 +78,7 @@ (into [:or] preds) (first preds))))) -(defn- update-where-clause - [hh-spec where] - (if-let [w (:where where)] - (assoc hh-spec :where w) - (dissoc hh-spec :where))) - -(defn view-check-template - "Receives a view-map and an action (insert/update/delete HoneySQL hash-map). - Returns a HoneySQL hash-map which will can be formatted as SQL to check if a - view needs to receive deltas for the action SQL." - [view-map action] - (let [{:keys [dummy-view dummy-args]} view-map - fc (determine-filter-clauses (:where dummy-view) dummy-args) - action-wc (format-action-wc-for-view view-map action) - view-map (append-arg-map view-map (:s fc))] ; we need this to compare *after* the check is run - (->> (-> dummy-view - (update-where-clause (vh/merge-where-clauses action-wc (:w fc))) - (merge (apply hh/select (:s fc)))) - (hash-map :view-map view-map :view-check)))) - -(defn prepare-checks-for-view-deltas - "Checks to see if an action has tables related to a view, and - if so builds the HoneySQL hash-map for the SQL needed. - Uses this hash-map as a key and conj's the view-map to the key's - value so as to avoid redundant delta-check querying." - [action confirmed-views view-map] - ;; Confirm if any of the tables in view-map are present in the action template: - (if (some (set (map first (vh/extract-tables action))) - (map first (:tables view-map))) - - ;; Then construct the check-template for this particular view. - (if-let [{:keys [view-check view-map]} (view-check-template view-map action)] - ;; We then use the view-check as an index and conj the - ;; view-map to it so as to avoid redundant checks. - (update-in confirmed-views [view-check] #(conj % view-map)) - confirmed-views) - confirmed-views)) - -(defn prepare-view-checks - "Prepares checks for a collection of views (view-maps) against a HoneySQL action - (insert/update/delete) hash-map. - - Returns a structure like so: - {{> views - (map #(check-view-args checked-results %)) - (remove nil?) - distinct)) - -(defn- do-view-pre-check - [db needs-deltas view-check] - ;; - ;; We have empty-select? if we have a view with no where predicate clauses-- - ;; so it will always require deltas if there are matching tables. - ;; - ;; empty-where comes about if we are inserting--we don't have any where predicate - ;; in the insert, of course, so we can't perform pre-checks reliably. - ;; When we do an insert we have to simply do the delta query regardless, for now. - ;; - (let [empty-select? (seq (remove nil? (:select (first view-check)))) - empty-where? (seq (remove #(or (nil? %) (= :and %)) (:where (first view-check))))] - (if (or (not empty-select?) (not empty-where?)) - (apply conj needs-deltas (last view-check)) ;; put them all in if we can't do pre-check. - (let [checked-results (do-check db (first view-check)) - ;; checks view args against checked result set - checked-views (check-all-view-args checked-results (last view-check))] - (if (seq checked-views) - (apply conj needs-deltas checked-views) - needs-deltas))))) - -(defn do-view-pre-checks - "Takes db, all views (view-maps) and the HoneySQL action (insert/update/delete) - hash-map. Returns view-maps for all the views which need to receive - delta updates after the action is performed. - - *This function should be called within a transaction before performing the - insert/update/delete action.*" - [db all-views action] - (let [view-checks (prepare-view-checks all-views action)] - (reduce #(do-view-pre-check db %1 %2) [] view-checks))) +;; DELTA CALCULATIONS (defn- calculate-delete-deltas [db view-map] @@ -181,10 +87,6 @@ (j/query db) (assoc view-map :delete-deltas))) -;; ------------------------------------------------------------------------------- -;; Handle inserts -;; ------------------------------------------------------------------------------- - (defn compute-delete-deltas-for-insert "Computes and returns a sequence of delete deltas for a single view and insert." [schema db view-map table record] @@ -227,8 +129,8 @@ {:views-with-deltas (doall (map #(assoc %1 :delete-deltas %2 :insert-deltas %3) views deletes inserts)) :result record*})) -;; Handles insert and calculation of insert (after insert) delta. (defn- insert-and-append-deltas! + "Handles insert and calculation of insert (after insert) delta." [schema db views action table pkey] (let [table (:insert-into action)] (reduce @@ -238,14 +140,12 @@ {:views-with-deltas [] :result-set []} (map #(compute-deltas-for-insert schema db views table %) (:values action))))) -;; ------------------------------------------------------------------------------- - -;; This is for insert deltas for non-insert updates. - -;;; Takes the HoneySQL map (at key :view) from the view-map and appends -;;; the appropriately-table-namespaced where clause which limits the -;;; view query to the previously inserted or updated records. (defn- calculate-insert-deltas + "This is for insert deltas for non-insert updates. + + Takes the HoneySQL map (at key :view) from the view-map and appends + the appropriately-table-namespaced where clause which limits the + view query to the previously inserted or updated records." [db action pkey-wc view-map] (let [action (assoc action :where pkey-wc) insert-delta-wc (format-action-wc-for-view view-map action) @@ -256,8 +156,8 @@ (update-in view-map [:insert-deltas] #(apply conj % deltas)) view-map))) -;; Helper to query the action's table for primary key and pull it out. (defn- get-action-row-key + "Helper to query the action's table for primary key and pull it out." [db pkey table action] (->> (:where action) (hsql/build :select pkey :from table :where) @@ -265,8 +165,8 @@ (j/query db) first pkey)) -;; Handles update and calculation of delete (before update) and insert (after update) deltas. (defn- update-and-append-deltas! + "Handles update and calculation of delete (before update) and insert (after update) deltas." [db views action table pkey] (let [views-pre (doall (map #(calculate-delete-deltas db %) views)) pkey-val (get-action-row-key db pkey table action) @@ -274,16 +174,16 @@ {:views-with-deltas (doall (map #(calculate-insert-deltas db action [:= pkey pkey-val] %) views-pre)) :result-set update})) -;; Handles deletion and calculation of delete (before update) delta. (defn- delete-and-append-deltas! + "Handles deletion and calculation of delete (before update) delta." [db views action table pkey] (let [views-pre (doall (map #(calculate-delete-deltas db %) views))] {:views-with-deltas views-pre :result-set (j/execute! db (hsql/format action))})) -;; Identifies which action--insert, update or delete--we are performing and dispatches appropriately. -;; Returns view-map with appropriate deltas appended. (defn perform-action-and-return-deltas + "Identifies which action--insert, update or delete--we are performing and dispatches appropriately. + Returns view-map with appropriate deltas appended." [schema db views action table pkey] (cond (:insert-into action) (insert-and-append-deltas! schema db views action table pkey) @@ -300,3 +200,63 @@ view (:view view-map)] (->> (update-in view [:where] #(:where (vh/merge-where-clauses action-wc %))) (assoc view-map :delete-deltas-map)))) + +(defn update-deltas-with-refresh-set + [refresh-set] + (fn [view-deltas] + (if (coll? view-deltas) + (map #(assoc % :refresh-set refresh-set) view-deltas) + [{:refresh-set refresh-set}]))) + +(defn calculate-refresh-sets + "For refresh-only views, calculates the refresh-set and adds it to the view's delta update collection." + [deltas db templates refresh-only-views] + (reduce + (fn [d {:keys [view-sig view] :as rov}] + (let [refresh-set (get (vdbl/initial-view db view-sig templates view) view-sig)] + (update-in d [view-sig] (update-deltas-with-refresh-set refresh-set)))) + deltas + refresh-only-views)) + +(defn format-deltas + "Removes extraneous data from view delta response collections." + [views-with-deltas] + (->> views-with-deltas + (map #(select-keys % [:view-sig :delete-deltas :insert-deltas :refresh-set])) + (group-by :view-sig))) + +(defn do-view-transaction + "Takes the following arguments: + schema - from edl.core/defschema + db - clojure.java.jdbc database connection + all-views - the current set of views (view-maps--see view-map fn docstring for + description) in memory for the database + action - the HoneySQL pre-SQL hash-map with parameters already interpolated. + templates - the mapping of view names (keywords) to SQL templates + (a.k.a. HoneySQL hash-map producing functions) + + The function will then perform the following sequence of actions, all run + within a transaction (with isolation serializable): + + 1) Create pre-check SQL for each view in the list. + 2) Run the pre-check SQL (or fail out based on some simple heuristics) to + identify if we want to send delta messages to the view's subscribers + (Note: this happens after the database action for *inserts only*). + 3) Run the database action (insert/action/delete). + 4) Calculate deltas based on the method described in section 5.4, \"Rule Generation\" + of the paper \"Deriving Production Rules for Incremental Rule Maintenance\" + by Stefano Ceri and Jennifer Widom (http://ilpubs.stanford.edu:8090/8/1/1991-4.pdf) + + The function returns a hash-map with :result-set and :new-deltas collection values. + :new-deltas contains :insert-deltas, :delete-deltas, and :refresh-set values, as well + as the original :view-sig the deltas apply to." + [schema db all-views action templates] + (j/with-db-transaction [t db :isolation :serializable] + (let [filtered-views (filterv #(vc/have-overlapping-tables? action (:view %)) all-views) + {full-refresh-views true normal-views nil} (group-by :refresh-only? filtered-views) + need-deltas (map #(generate-view-delta-map % action) normal-views) + table (-> action vh/extract-tables ffirst) + pkey (get-primary-key schema table) + {:keys [views-with-deltas result-set]} (perform-action-and-return-deltas schema t need-deltas action table pkey) + deltas (calculate-refresh-sets (format-deltas views-with-deltas) t templates full-refresh-views)] + {:new-deltas deltas :result-set result-set}))) diff --git a/src/views/db/load.clj b/src/views/db/load.clj index 30e5003..7f3c4ae 100644 --- a/src/views/db/load.clj +++ b/src/views/db/load.clj @@ -1,5 +1,6 @@ (ns views.db.load (:require + [clojure.tools.logging :refer [debug info warn error]] [clojure.java.jdbc :as j] [honeysql.core :as hsql])) diff --git a/src/views/subscriptions.clj b/src/views/subscriptions.clj index 3203210..5bb53e8 100644 --- a/src/views/subscriptions.clj +++ b/src/views/subscriptions.clj @@ -1,6 +1,6 @@ (ns views.subscriptions (:require - [views.db.core :as vdb])) + [views.db.deltas :as vd])) ;; ;; {namespace {[:view-sig 1 "arg2"] {:subscriptions [1 2 3 4 ... ] :view-data {:view ...}}}} @@ -22,7 +22,7 @@ (fn [svs] (-> svs (update-in [namespace view-sig :subscriptions] (add-subscriber-key subscriber-key)) - (assoc-in [namespace view-sig :view-data] (vdb/view-map (get-in templates [(first view-sig) :fn]) view-sig))))) + (assoc-in [namespace view-sig :view-data] (vd/view-map (get-in templates [(first view-sig) :fn]) view-sig))))) (defn add-subscription! ([view-sig templates subscriber-key] diff --git a/test/views/db/deltas_test.clj b/test/views/db/deltas_test.clj index f0456e8..e065c4d 100644 --- a/test/views/db/deltas_test.clj +++ b/test/views/db/deltas_test.clj @@ -1,88 +1,77 @@ (ns views.db.deltas-test (:require - [clojure.test :refer [deftest is run-tests]] + [clojure.test :refer [use-fixtures deftest is]] [honeysql.core :as hsql] [honeysql.helpers :as hh] - [views.fixtures :as vf] + [views.fixtures :as vf :refer [vschema sql-ts]] [views.db.core :as vdb] - [views.db.deltas :as vdbd] - [views.base-subscribed-views :as bsv]) - (:import - [views.base_subscribed_views BaseSubscribedViews])) + [views.db.deltas :as vd])) -(defn join-test-template - [id val3] - (-> (hh/select :f.id :f.val1 :f.val2 :b.val1) - (hh/from [:foo :f]) - (hh/join [:bar :b] [:= :b.id :f.b_id]) - (hh/left-join [:baz :ba] [:= :ba.id :b.ba_id]) - (hh/right-join [:qux :q] [:= :q.id :ba.q_id]) - (hh/where [:= :f.id id] [:= :f.val3 val3] [:= :f.val2 "constant"]))) +(defn dvt-helper + ([all-views action] (dvt-helper all-views action vf/templates)) + ([all-views action templates] + (vd/do-view-transaction vschema vf/db all-views action templates))) -(defn no-where-view-template - [] - (-> (hh/select :f.id :f.val1 :f.val2) - (hh/from [:foo :f]))) +(use-fixtures :each vf/database-fixtures!) -(defn bar-template - [id] - (-> (hh/select :b.id :b.val1) - (hh/from [:bar :b]) - (hh/where [:= :val2 "some constant"] - [:= :id id]))) +(deftest builds-view-map + (let [{:keys [view-sig view refresh-only?]} (vd/view-map vf/users-tmpl [:users])] + (is (= view-sig [:users])) + (is (= view {:from [:users], :select [:id :name :created_on]})) + (is (nil? refresh-only?)))) -(defn unrelated-template - [id] - (-> (hh/select :u.id :u.val1) - (hh/from :unrelated - (hh/where [:= :val "some constant"] - [:= :id id])))) +(defn non-nil-values-for-keys? + [hm keys] + (every? #(% hm) keys)) -(defn update-bar-template - [val1 wc] - (-> (hh/update :bar) - (hh/values {:val1 val1}) - (hh/where wc))) +(deftest calculates-insert-deltas + (let [views [(vd/view-map vf/users-tmpl [:users])] + user-args {:name "Test user" :created_on (sql-ts)} + insert (hsql/build :insert-into :users :values [user-args]) + {:keys [new-deltas result-set]} (dvt-helper views insert) + insert-delta (first (:insert-deltas (first (get new-deltas [:users]))))] -(deftest constructs-view-check-template - (let [update-bar (update-bar-template "foo" [:= :id 123]) - vm (vdb/view-map join-test-template [:join-test 1 "foo"]) - check-template (:view-check (vdbd/view-check-template vm update-bar))] - (is (= (set (:select check-template)) #{:f.id :f.val3})) - (is (= (set (rest (:where check-template))) #{[:= :f.val2 "constant"] [:= :b.id 123]})))) + ;; Result set + (is (not (nil? (:id (first result-set))))) + (is (= user-args (dissoc (first result-set) :id))) -(deftest view-check-template-generates-proper-sql - (let [update-bar (update-bar-template "foo" [:= :id 123]) - vm (vdb/view-map join-test-template [:join-test 1 "foo"]) - check-template (:view-check (vdbd/view-check-template vm update-bar))] - (is (= (hsql/format check-template) - ["SELECT f.id, f.val3 FROM foo f INNER JOIN bar b ON b.id = f.b_id LEFT JOIN baz ba ON ba.id = b.ba_id RIGHT JOIN qux q ON q.id = ba.q_id WHERE (b.id = 123 AND f.val2 = ?)" "constant"])))) + ;; Deltas + (is (= (:name user-args) (:name insert-delta))) + (is (= (:created_on user-args) (:created_on insert-delta))) + (is (non-nil-values-for-keys? insert-delta (-> views first :view :select))))) -(deftest creates-collection-of-views-to-check - (let [views [(vdb/view-map no-where-view-template [:no-where]) ; no :bar - (vdb/view-map no-where-view-template [:no-where]) ; no :bar - (vdb/view-map bar-template [:bar 1]) ; has :bar - (vdb/view-map unrelated-template [:unrelated 2]) ; no :bar - (vdb/view-map join-test-template [:join-test 1 "foo"]) ; has :bar - (vdb/view-map join-test-template [:join-test 2 "bar"])] ; has :bar - update-bar (update-bar-template "foo" [:= :id 123]) - checked-views (vdbd/prepare-view-checks views update-bar)] +(deftest calculates-delete-deltas + (let [views [(vd/view-map vf/users-tmpl [:users])] + user-args {:name "Test user" :created_on (sql-ts)} + user (vf/view-action! (hsql/build :insert-into :users :values [user-args])) + delete (hsql/build :delete-from :users :where [:= :name (:name user-args)]) + {:keys [new-deltas result-set]} (dvt-helper views delete) + delete-delta (first (:delete-deltas (first (get new-deltas [:users]))))] - ;; It should return one check for the bar-template above, - ;; and 1 for *both* the joint-test-templates. - (is (= (count checked-views) 2)))) + ;; Deltas + (is (= (:name user-args) (:name delete-delta))) + (is (= (:created_on user-args) (:created_on delete-delta))) + (is (non-nil-values-for-keys? delete-delta (-> views first :view :select))))) -;; What is this for? -(def left-join-example (hsql/build :select [:R.a :S.C] :from :R :left-join [:S [:= :R.B :S.B]] :where [:!= :S.C 20])) +(deftest calculates-update-deltas + (let [views [(vd/view-map vf/users-tmpl [:users])] + user-args {:name "Test user" :created_on (sql-ts)} + user (vf/view-action! (hsql/build :insert-into :users :values [user-args])) + new-name "new name!" + update (hsql/build :update :users :set {:name new-name} :where [:= :name (:name user-args)]) + {:keys [new-deltas result-set]} (dvt-helper views update) + {:keys [insert-deltas delete-deltas]} (first (get new-deltas [:users]))] -;; (deftest notes-view-map-as-no-delta-calc -;; (let [tmpl (with-meta vf/users-tmpl {:bulk-update? true})] -;; (is (:bulk-update? (vdb/view-map tmpl [:users]))))) + ;; Deltas + (is (= (:name user-args) (:name (first delete-deltas)))) + (is (= new-name (:name (first insert-deltas)))))) -;; (defschema schema vf/db "public") - -;; (deftest sends-entire-view-on-every-update-with-bulk-update -;; (let [tmpl (with-meta vf/users-tmpl {:bulk-update? true}) -;; vm (vdb/view-map tmpl [:users]) -;; bsv (BaseSubscribedViews. vf/db +(deftest does-not-calculate-deltas-for-unrelated-views + (let [views [(vd/view-map vf/users-tmpl [:users]) + (vd/view-map vf/all-comments-tmpl [:all-comments])] + user-args {:name "Test user" :created_on (sql-ts)} + insert (hsql/build :insert-into :users :values [user-args]) + {:keys [new-deltas result-set]} (dvt-helper views insert)] +;; (is (= (count (insert-deltas new-deltas) 1)) + (is (nil? (get new-deltas [:all-comments]))))) diff --git a/test/views/fixtures.clj b/test/views/fixtures.clj index 3661335..f55c8dc 100644 --- a/test/views/fixtures.clj +++ b/test/views/fixtures.clj @@ -3,6 +3,7 @@ [environ.core :as e] [clojure.java.jdbc :as j] [honeysql.core :as hsql] + [edl.core :refer [defschema]] [clojure.data.generators :as dg])) (defn sql-ts @@ -15,6 +16,8 @@ :user (get :views-test-user e/env "views_user") :password (get :views-test-ppassword e/env "password")}) +(defschema vschema db "public") + (defn clean-tables! [tables] (doseq [t (map name tables)] @@ -22,19 +25,42 @@ (defn database-fixtures! [f] - (clean-tables! [:posts :users]) + (clean-tables! [:posts :users :comments]) (f)) +(defn rand-str + [l] + (dg/string #(rand-nth (seq "abcdefghijklmnopqrstuwvxyz ")) l)) + +(defn view-query + [view] + (j/query db (hsql/format view))) + +(defn view-action! + [action] + (j/execute! db (hsql/format action))) + (defn user-fixture! [name] - (j/execute! db (hsql/format (hsql/build :insert-into :users :values [{:name name :created_on (sql-ts)}])))) + (view-action! (hsql/build :insert-into :users :values [{:name name :created_on (sql-ts)}]))) (defn gen-n-users! [n] - (dotimes [n n] - (user-fixture! (dg/string #(rand-nth (seq "abcdefghijklmnopqrstuwvxyz"))))) + (dotimes [n n] (user-fixture! (rand-str 10))) (j/query db ["SELECT * FROM users"])) +(defn insert-post-tmpl + [uid title body] + (hsql/build :insert-into :posts :values [{:user_id uid :title title :body body :created_on (sql-ts)}])) + +(defn post-fixture! + [uid title body] + (view-action! (insert-post-tmpl uid title body))) + +(defn gen-n-posts-for-user! + [n uid] + (dotimes [n n] (post-fixture! uid (rand-str 20) (rand-str 100)))) + (defn users-tmpl [] (hsql/build :select [:id :name :created_on] :from :users)) @@ -46,6 +72,17 @@ :join [[:users :u][:= :u.id :p.user_id]] :where [:= :p.user_id user_id])) +(defn users-posts-tmpl + [] + (hsql/build :select [[:u.id :user_id] :u.name :p.id :p.title :p.body :p.created_on] + :from {:users :u} + :left-join [[:posts :p][:= :u.id :p.user_id]])) + +(defn all-comments-tmpl + [] + (hsql/build :select [:id :body :created_on] :from {:comments :c})) + (def templates - {:users {:fn #'users-tmpl} - :user-posts {:fn #'user-posts-tmpl}}) + {:users {:fn #'users-tmpl} + :user-posts {:fn #'user-posts-tmpl} + :all-comments {:fn #'all-comments-tmpl}}) diff --git a/test/views/test_db.sql b/test/views/test_db.sql index 51f0bae..e30746c 100644 --- a/test/views/test_db.sql +++ b/test/views/test_db.sql @@ -1,12 +1,18 @@ CREATE ROLE views_user LOGIN PASSWORD 'password'; CREATE DATABASE views_test OWNER views_user; \c postgresql://localhost/views_test; -CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL, created_on DATE NOT NULL); +CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL, created_on TIMESTAMP NOT NULL); CREATE TABLE posts (id SERIAL PRIMARY KEY, title TEXT NOT NULL, body TEXT NOT NULL, - created_on DATE NOT NULL, + created_on TIMESTAMP NOT NULL, user_id INTEGER NOT NULL, FOREIGN KEY (user_id) REFERENCES users(id)); +CREATE TABLE comments (id SERIAL PRIMARY KEY, + body TEXT NOT NULL, + created_on TIMESTAMP NOT NULL, + post_id INTEGER NOT NULL, + FOREIGN KEY (post_id) REFERENCES posts(id)); ALTER TABLE users OWNER TO views_user; ALTER TABLE posts OWNER TO views_user; +ALTER TABLE comments OWNER TO views_user;