bump to next snapshot version for future development

needed for lein release usage

.gitignore

@@ -1,12 +1,18 @@
+.DS_Store
 /target
 /classes
 /checkouts
+/out
 pom.xml
 pom.xml.asc
 *.jar
 *.class
-/.lein-*
+.lein-*
-/.nrepl-port
+.nrepl-port
-*~
+/*.project
-*.bk
+/*.classpath
-.idea
+/.settings/
+*.iml
+*.ipr
+*.iws
+.idea

LICENSE

@@ -1,214 +1,21 @@
-THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC
+The MIT License (MIT)
-LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM
-CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT.
 
-1. DEFINITIONS
+Copyright (c) 2015 Kira Inc.
 
-"Contribution" means:
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-a) in the case of the initial Contributor, the initial code and
+The above copyright notice and this permission notice shall be included in
-documentation distributed under this Agreement, and
+all copies or substantial portions of the Software.
 
-b) in the case of each subsequent Contributor:
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-i) changes to the Program, and
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-ii) additions to the Program;
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-where such changes and/or additions to the Program originate from and are
+THE SOFTWARE.
-distributed by that particular Contributor. A Contribution 'originates' from
-a Contributor if it was added to the Program by such Contributor itself or
-anyone acting on such Contributor's behalf. Contributions do not include
-additions to the Program which: (i) are separate modules of software
-distributed in conjunction with the Program under their own license
-agreement, and (ii) are not derivative works of the Program.
-
-"Contributor" means any person or entity that distributes the Program.
-
-"Licensed Patents" mean patent claims licensable by a Contributor which are
-necessarily infringed by the use or sale of its Contribution alone or when
-combined with the Program.
-
-"Program" means the Contributions distributed in accordance with this
-Agreement.
-
-"Recipient" means anyone who receives the Program under this Agreement,
-including all Contributors.
-
-2. GRANT OF RIGHTS
-
-a) Subject to the terms of this Agreement, each Contributor hereby grants
-Recipient a non-exclusive, worldwide, royalty-free copyright license to
-reproduce, prepare derivative works of, publicly display, publicly perform,
-distribute and sublicense the Contribution of such Contributor, if any, and
-such derivative works, in source code and object code form.
-
-b) Subject to the terms of this Agreement, each Contributor hereby grants
-Recipient a non-exclusive, worldwide, royalty-free patent license under
-Licensed Patents to make, use, sell, offer to sell, import and otherwise
-transfer the Contribution of such Contributor, if any, in source code and
-object code form.  This patent license shall apply to the combination of the
-Contribution and the Program if, at the time the Contribution is added by the
-Contributor, such addition of the Contribution causes such combination to be
-covered by the Licensed Patents. The patent license shall not apply to any
-other combinations which include the Contribution. No hardware per se is
-licensed hereunder.
-
-c) Recipient understands that although each Contributor grants the licenses
-to its Contributions set forth herein, no assurances are provided by any
-Contributor that the Program does not infringe the patent or other
-intellectual property rights of any other entity. Each Contributor disclaims
-any liability to Recipient for claims brought by any other entity based on
-infringement of intellectual property rights or otherwise. As a condition to
-exercising the rights and licenses granted hereunder, each Recipient hereby
-assumes sole responsibility to secure any other intellectual property rights
-needed, if any. For example, if a third party patent license is required to
-allow Recipient to distribute the Program, it is Recipient's responsibility
-to acquire that license before distributing the Program.
-
-d) Each Contributor represents that to its knowledge it has sufficient
-copyright rights in its Contribution, if any, to grant the copyright license
-set forth in this Agreement.
-
-3. REQUIREMENTS
-
-A Contributor may choose to distribute the Program in object code form under
-its own license agreement, provided that:
-
-a) it complies with the terms and conditions of this Agreement; and
-
-b) its license agreement:
-
-i) effectively disclaims on behalf of all Contributors all warranties and
-conditions, express and implied, including warranties or conditions of title
-and non-infringement, and implied warranties or conditions of merchantability
-and fitness for a particular purpose;
-
-ii) effectively excludes on behalf of all Contributors all liability for
-damages, including direct, indirect, special, incidental and consequential
-damages, such as lost profits;
-
-iii) states that any provisions which differ from this Agreement are offered
-by that Contributor alone and not by any other party; and
-
-iv) states that source code for the Program is available from such
-Contributor, and informs licensees how to obtain it in a reasonable manner on
-or through a medium customarily used for software exchange.
-
-When the Program is made available in source code form:
-
-a) it must be made available under this Agreement; and
-
-b) a copy of this Agreement must be included with each copy of the Program.
-
-Contributors may not remove or alter any copyright notices contained within
-the Program.
-
-Each Contributor must identify itself as the originator of its Contribution,
-if any, in a manner that reasonably allows subsequent Recipients to identify
-the originator of the Contribution.
-
-4. COMMERCIAL DISTRIBUTION
-
-Commercial distributors of software may accept certain responsibilities with
-respect to end users, business partners and the like. While this license is
-intended to facilitate the commercial use of the Program, the Contributor who
-includes the Program in a commercial product offering should do so in a
-manner which does not create potential liability for other Contributors.
-Therefore, if a Contributor includes the Program in a commercial product
-offering, such Contributor ("Commercial Contributor") hereby agrees to defend
-and indemnify every other Contributor ("Indemnified Contributor") against any
-losses, damages and costs (collectively "Losses") arising from claims,
-lawsuits and other legal actions brought by a third party against the
-Indemnified Contributor to the extent caused by the acts or omissions of such
-Commercial Contributor in connection with its distribution of the Program in
-a commercial product offering.  The obligations in this section do not apply
-to any claims or Losses relating to any actual or alleged intellectual
-property infringement. In order to qualify, an Indemnified Contributor must:
-a) promptly notify the Commercial Contributor in writing of such claim, and
-b) allow the Commercial Contributor tocontrol, and cooperate with the
-Commercial Contributor in, the defense and any related settlement
-negotiations. The Indemnified Contributor may participate in any such claim
-at its own expense.
-
-For example, a Contributor might include the Program in a commercial product
-offering, Product X. That Contributor is then a Commercial Contributor. If
-that Commercial Contributor then makes performance claims, or offers
-warranties related to Product X, those performance claims and warranties are
-such Commercial Contributor's responsibility alone. Under this section, the
-Commercial Contributor would have to defend claims against the other
-Contributors related to those performance claims and warranties, and if a
-court requires any other Contributor to pay any damages as a result, the
-Commercial Contributor must pay those damages.
-
-5. NO WARRANTY
-
-EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON
-AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER
-EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR
-CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A
-PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the
-appropriateness of using and distributing the Program and assumes all risks
-associated with its exercise of rights under this Agreement , including but
-not limited to the risks and costs of program errors, compliance with
-applicable laws, damage to or loss of data, programs or equipment, and
-unavailability or interruption of operations.
-
-6. DISCLAIMER OF LIABILITY
-
-EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY
-CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION
-LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE
-EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY
-OF SUCH DAMAGES.
-
-7. GENERAL
-
-If any provision of this Agreement is invalid or unenforceable under
-applicable law, it shall not affect the validity or enforceability of the
-remainder of the terms of this Agreement, and without further action by the
-parties hereto, such provision shall be reformed to the minimum extent
-necessary to make such provision valid and enforceable.
-
-If Recipient institutes patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Program itself
-(excluding combinations of the Program with other software or hardware)
-infringes such Recipient's patent(s), then such Recipient's rights granted
-under Section 2(b) shall terminate as of the date such litigation is filed.
-
-All Recipient's rights under this Agreement shall terminate if it fails to
-comply with any of the material terms or conditions of this Agreement and
-does not cure such failure in a reasonable period of time after becoming
-aware of such noncompliance. If all Recipient's rights under this Agreement
-terminate, Recipient agrees to cease use and distribution of the Program as
-soon as reasonably practicable. However, Recipient's obligations under this
-Agreement and any licenses granted by Recipient relating to the Program shall
-continue and survive.
-
-Everyone is permitted to copy and distribute copies of this Agreement, but in
-order to avoid inconsistency the Agreement is copyrighted and may only be
-modified in the following manner. The Agreement Steward reserves the right to
-publish new versions (including revisions) of this Agreement from time to
-time. No one other than the Agreement Steward has the right to modify this
-Agreement. The Eclipse Foundation is the initial Agreement Steward. The
-Eclipse Foundation may assign the responsibility to serve as the Agreement
-Steward to a suitable separate entity. Each new version of the Agreement will
-be given a distinguishing version number. The Program (including
-Contributions) may always be distributed subject to the version of the
-Agreement under which it was received. In addition, after a new version of
-the Agreement is published, Contributor may elect to distribute the Program
-(including its Contributions) under the new version. Except as expressly
-stated in Sections 2(a) and 2(b) above, Recipient receives no rights or
-licenses to the intellectual property of any Contributor under this
-Agreement, whether expressly, by implication, estoppel or otherwise. All
-rights in the Program not expressly granted under this Agreement are
-reserved.
-
-This Agreement is governed by the laws of the State of Washington and the
-intellectual property laws of the United States of America. No party to this
-Agreement will bring a legal action under this Agreement more than one year
-after the cause of action arose. Each party waives its rights to a jury trial
-in any resulting litigation.

NOTES.md

@@ -0,0 +1,19 @@
+# Notes
+
+## Note 1
+
+There is one hash stored per view signature. If a view is refreshed, a hash of
+its data is checked against the previous data and if they match it is not sent
+out. Otherwise it is sent out and the hash is replaced. On subscription, we
+also store a hash of the view data, but only if it doesn't exist. This is important
+because if we always stored it, the following situation would be a problem.
+
+
+   u1 - subscribes v1, hash is stored
+   p1 - updates data requiring v1 to be updated
+   u2 - subscribes v1, hash is stored
+   t1 - update thread runs and decides v1 needs to be updated because of the hint
+        supplied by p1, however, the hash is now the same because of u2 and no
+        refresh is sent out to u1.
+
+        

README.md

@@ -1,41 +1,739 @@
 # views
 
-Eventually consistent external materialized views for SQL databases.
+Eventually consistent external materialized views.
 
-## Design
+Also see these plugin libraries which allow you to use a views system
+in a number of really great ways in your applications:
+
+* [views.sql](https://github.com/gered/views.sql)
+* [views.honeysql](https://github.com/gered/views.honeysql)
+* [views.reagent](https://github.com/gered/views.reagent)
+
+## Leiningen
+
+[![](https://clojars.org/net.gered/views/latest-version.svg)](https://clojars.org/net.gered/views)
+
+## This is a fork!
+I'm keeping this as a separate fork for now because I've made some 
+breaking or otherwise significant changes from the original, some of 
+which are based simply on personal preferences. I simply haven't felt 
+it's quite right to submit a pull request because of some of these 
+types of changes. Perhaps in the future.
+
+I definitely cannot take credit for the original idea behind this 
+library or the core implementation or design of it. Definitely keep an 
+eye on the [original repository][1] which is maintained by Kira Inc.
+
+[1]: https://github.com/kirasystems/views
+
+
+## Basic Concepts
+
+The views library allows you to manage a **view system** which is a 
+collection of **views** and a list of **subscribers** to those views. 
+Subscribers will get sent **view refreshes** in realtime when the data 
+represented by the views they are subscribed to changes. Relevant 
+changes are found through the use of **hints** which are added to the 
+view system by anything that is actually changing the data at the 
+instant it is changed.
+
+A **view** is similar in concept to a [materialized view][2], though in
+practice it may not actually keep a copy of the underlying data 
+represented by the view and instead just keep a copy of a query or the 
+location of where the data can be retrieved from when it is needed 
+(e.g. when view refrehses need to be sent out).
+
+[2]: https://en.wikipedia.org/wiki/Materialized_view
+
+A view is represented by the protocol `views.protocols/IView`:
+
+```clj
+(defprotocol IView
+  (data [this namespace parameters])
+  (relevant? [this namespace parameters hints])
+  (id [this]))
+```
+
+`id` simply returns a unique identifier for this view. `data` returns a
+copy of the underlying data represented by this view. `relevant?` 
+determines if a collection of hints are relevant to the view and is 
+called by the view system whenever new hints are received to determine 
+if view refreshes need to be sent out for this view.
+
+A **hint** is a map of the form:
+
+```clj
+{:namespace ...
+ :type ...
+ :hint ...}
+```
+
+`:type` represents the type of view (e.g. `:sql-table-name`) and is 
+defined by the view implementation that this hint is intended for. 
+`:hint` is the actual hint information itself and it's contents will 
+differ depending on the type of view it is intended for. As an example,
+for a SQL view it may be a list of database table names. 
+
+**Namespaces** can be used to isolate multiple sets of the same type of
+data being represented by the views within the view system. As an 
+example, for SQL views a namespace could be used to represent the 
+database to connect to if your system is comprised of multiple similar 
+databases. A view is not specifically tied to a namespace, however the 
+hints processed by the view system are only relevant for the namespace 
+specified in the hint.
+
+When a view's `relevant?` check is determining if any given hint is 
+relevant or not, it will compare all the properties of a hint, 
+including the namespace and type to ensure that view refreshes aren't 
+issued incorrectly or too frequently.
+
+**Subscribers** can be registered within the view system. A 
+subscription can be created within the view system by specifying the 
+view to subscribe to, identified by it's view ID, and also a namespace 
+and any parameters that the view might take. These 3 properties go 
+together to form a **view signature** or **view sig**. A view sig is 
+represented by a map:
+
+```clj
+{:namespace ...
+ :view-id ...
+ :parameters ...}
+```
+
+Subscriptions are considered unique for a subscriber based on all 3 of
+these properties combined. As such a subscriber can have multiple 
+concurrent subscriptions to the same view if the namespace and/or 
+parameters are different for all of them.
+
+A subscriber is uniquely identified by it's **subscriber key**. Common
+subscriber key values include user ID's, Session ID's, or other 
+identifiers like client ID's used in libraries like Sente for websocket
+connections.
+
+When hints are processed by the view system and found to be relevant 
+for any of the views (through the use of the `relevant?` check 
+mentioned earlier), **view refreshes** are sent out to all of the 
+subscribers of the view. Up-to-date data for the view is retrieved via 
+the view's `data` function and then sent out.
+
+Whenever data is refreshed a hash is kept and is compared to on each 
+refresh to make sure that we don't send out another refresh if the data
+is unchanged from the last refresh sent.
 
-TODO
 
 ## Usage
 
-TODO
+To explain basic usage of the views library, we'll walk through an
+example building up a simple system so you can see how it works
+interactively.
 
-## Testing
+To begin, we'll need to use functions from the `views.core` namespace.
 
-You will need to set up the test db to run the tests:
+```clj
-
+(require '[views.core :as views])
-```bash
-$ psql -Upostgres < test/views/test_db.sql
-CREATE ROLE
-CREATE DATABASE
-$
 ```
 
-This will create a role `views_user` and a database owned by that user called `views_test`.
+### View System Initialization
 
-(You can change the database settings if you'd like by editing that file and checking the config in `test/views/fixtures.clj`.)
+We first need to create the view system. This will be kept in an atom
+and will be passed around to the different views library functions also
+*as an atom* as the views system needs to maintain it's own internal
+state.
 
-Then, to run all tests:
+```clj
-
+(def view-system (atom {}))
-```bash
-$ lein with-profile test test
 ```
 
+For a fully working view system, we need to also provide a function
+that will be used to send view refreshes to subscribers. For now we'll 
+just print view refreshes out in the REPL, but in a real system you'd
+probably want this to send them to a connected Websocket client, or out
+over some kind of distributed messaging service, etc.
+
+```clj
+(defn send-fn
+  [subscriber-key [view-sig view-data]]
+  (println "view refresh" subscriber-key view-sig view-data))
+```
+
+Now we're ready to actually create the view syem. To do this we call 
+`init!` which takes a set of options. We provide our send function 
+above using the `:send-fn` option. For a description of all the options
+available, see `views.core/default-options`.
+
+```clj
+(views/init! view-system {:send-fn send-fn})
+```
+
+At this point, the view system is ready.
+
+Right now there are some background threads running, one of which is
+the *refresh watcher* which handles incoming hints and checks them for
+relevancy. When relevant hints are found, view refresh requests are
+dispatched to one or more *refresh worker* threads which actually
+perform the work of retrieving updated view data and sending it off to
+subscribers.
+
+But now we need to talk about setting up some views, as we have none in
+our view system.
+
+### Adding Views
+
+For demonstration purposes, we'll set up views for an in-memory
+datastore:
+
+```clj
+(def memory-datastore
+  (atom {:a {:foo 1 
+             :bar 200 
+             :baz [1 2 3]}
+         :b {:foo 2 
+             :bar 300 
+             :baz [2 3 4]}}))
+```
+
+To retrieve or modify data within this memory datastore, we'd likely
+want to use a path made up of keywords, e.g. `[:a :bar]` would
+correspond with the value `200`, and `[:b :baz 2]` with the value `4` 
+using the initial data defined above.
+
+So, let's create a `MemoryView`:
+
+```clj
+(require '[views.protocols :refer [IView]])
+
+(def memory-view-hint-type :ks-path)
+
+(defrecord MemoryView [id ks]
+  IView
+  (id [_] id)
+  (data [_ namespace parameters]
+    (get-in @memory-datastore 
+            (-> [namespace]
+                (into ks)
+                (into parameters))))
+  (relevant? [_ namespace parameters hints]
+    (some #(and (= namespace (:namespace %))
+                (= ks (:hint %))
+                (= memory-view-hint-type (:type %)))
+          hints)))
+```
+
+Nothing particularly special here, `data` simply returns a value from 
+`memory-datastore` using a path made by combining `namespace` with a 
+sequence of keywords `ks` and then finally adding `parameters` (which 
+is a collection of parameters) to the end of the path.
+
+Note that with this method of referencing data within 
+`memory-datastore`, the keys `:a` and `:b` are being used as 
+namespaces.
+
+`relevant?` simply compares all 3 values of each of the hints passed in
+to make sure they all match. `memory-view-hint-type` is, as it's name
+implies, a value that is used to identify hints as being those intended
+for memory views and not for, e.g. SQL views (if we had a view system 
+with multiple different types of views in it). The function returns 
+true if at least one of the passed in hints was found to be relevant.
+
+Now we can add some views to our view system:
+
+```clj
+(views/add-views!
+  view-system
+  [(MemoryView. :foo [:foo])
+   (MemoryView. :bar [:bar])
+   (MemoryView. :baz [:baz])])
+```
+
+We now have 3 views, `:foo`, `:bar` and `:baz` which each refer to data
+under that same path. Note that these views do not define a namespace.
+That is for subscribers to specify when they register a subscription. 
+As well, code that updates `memory-datastore` will create hints for the
+view system as we'll soon see, and at that time it will include a 
+namespace in any created hints.
+
+> Most applications will probably want to just pass in a list of views
+> via `views.core/init!` through the `:views` option. However, there is
+> nothing wrong with using `add-views!` like this if you prefer or if 
+> you simply need to change views on the fly.
+> 
+> Keep in mind though that adding views via `add-views!` will replace 
+> existing views in the view system with the same ID. Take care when 
+> doing this if there is the possibility that there are existing 
+> subscribers to views that are being replaced!
+
+### Subscribing to Views
+
+As mentioned previously, view subscriptions are keyed by a **view
+signature** or view sig, which we can create using a helper function if
+we wish:
+
+```clj
+(views/->view-sig :a :foo [])
+=> {:namespace :a, :view-id :foo, :parameters []}
+```
+
+We create a subscription by calling `views.core/subscribe!`. For this
+demonstration, we'll simply make up a subscriber key. The last argument
+is where we could pass in some application/user context data that would
+be helpful to use when doing subscription authorization (which we'll 
+discuss later and just ignore for now). For now, we'll just pass in 
+`nil` context.
+
+```clj
+(views/subscribe! 
+  view-system                        ; view system atom
+  (views/->view-sig :a :foo [])      ; view sig of the view to subscribe to
+  123                                ; subscriber key
+  nil)                               ; context
+```
+
+`subscribe!` returns a `future` which will be realized when the 
+subscription finishes. Whenever a new subscription is added, the 
+subscriber is sent an initial set of data for the view. This view 
+refresh is done in a separate thread via a `future`.
+
+We can see that a view refresh was sent out as a result of this 
+subscription as our `send-fn` function from before was called and the 
+following output should have appeared
+
+```
+view refresh 123 {:view-id :foo, :parameters []} 1
+```
+
+right away after the call to `subscribe!`. The `1` at the end 
+corresponds to the data in `memory-datastore` under the path 
+`[:a :foo]`.
+
+> Note that an initial view refresh is **always** sent out to the 
+> subscriber when a subscription is first created. This happens even 
+> if the view data has not changed since the last refresh for this view
+> occurred, as obviously the new subscriber was not part of that 
+> refresh.
+
+### Hints and View Refreshes
+
+Adding hints to the view system triggers refreshes of views for which 
+they are relevant towards. Our application code that changes data which
+these views are based on needs to have a way of adding views to the 
+view system.
+
+#### Hints
+
+As mentioned previously, a hint is simply a map that contains a 
+namespace, a type and some data that will differ based on the types of
+views in the view system. There is a helper function to create this 
+map:
+
+```clj
+(views/hint :a [:foo] memory-view-hint-type)
+=> {:namespace :a, :hint [:foo], :type :ks-path}
+```
+
+Generally speaking the `:type` value will be the same for all hints 
+which are intended for the same types of views. For example, all of our
+`MemoryView` views expect the type to be `:ks-path`, because the
+`:hint` values they expect to compare against are all keyword paths.
+
+#### Adding Hints to the View System
+
+There are two main ways to do this:
+
+1. Queue hints which will be picked up by the refresh watcher thread on
+   a regular interval (set by the option `:refresh-interval`).
+2. Immediately trigger a refresh for a list of hints.
+
+Using option 2 all the time generally does result in much more
+responsive feeling system from the user's perspective. But you should
+also consider just how frequently your code could end up triggering
+refreshes.
+
+Queueing hints as in option 1 will help to guard against duplicate 
+hints triggering excessive view refreshes as duplicate hints added to 
+the queue are dropped. But queued hints are not processed until the 
+refresh watcher thread runs at the next `:refresh-interval`, so you 
+lose some responsiveness by going this route.
+
+There are more factors to consider in addition to all of this though. 
+As hints are processed, they are internally turned into view refresh 
+requests and dispatched to the refresh worker threads by adding them to
+an internal queue. This refresh queue also drops duplicate requests, 
+but only if there is a backlog of refresh requests waiting in the queue
+(which would happen if some views are taking too long to refresh, e.g. 
+slow SQL queries, overloaded server/network, not enough worker threads,
+etc). If the worker threads are able to process refresh requests very 
+quickly, then the internal queue will usually be empty or near-empty 
+and some or all duplicate refresh requests might make it through.
+
+Also keep in mind that hashes of view data are computed and then 
+compared each time a view refresh is about to be sent out, and while 
+the underlying view data must still be retrieved to compute this hash 
+each time a refresh request is processed, a view refresh will not 
+actually be sent out to the subscribers if the data is found to be 
+unchanged since the last refresh.
+
+Ultimately there isn't really a right or wrong answer as to which 
+method you choose. Generally speaking it will usually make the most 
+sense to default to option 2 for most actions that need to add hints to
+the view system. This will generally result in a more responsive 
+system. But you'll want to continually evaluate whether some actions 
+should possibly be switched over to queue up hints instead.
+
+#### Option 1: Queueing Hints
+
+Use `queue-hints!` and pass in a collection of hints. They will be 
+added to the queue and the refresh watcher thread will process them on 
+the next refresh interval.
+
+```clj
+(views/queue-hints! 
+  view-system 
+  [(views/hint :a [:foo] memory-view-hint-type)])
+```
+
+#### Option 2: Immediately Trigger Refreshes From Hints
+
+Use `refresh-views!` and pass in a collection of hints. They will be 
+processed immediately and refresh requests will be dispatched for all 
+views for which there were relevant hints (and subscribers) for.
+
+```clj
+(views/refresh-views! 
+  view-system 
+  [(views/hint :a [:foo] memory-view-hint-type)])
+```
+
+#### But Wait -- View Data Must Be Changed First!
+
+If you were following along and tried the above examples out, you would
+have noticed that our `send-fn` function was never called. As mentioned
+previously, each time a view refresh is processed a hash is taken of 
+the data and compared against the previous refresh's hash. Only if the 
+data is found to have been changed is a refresh sent out.
+
+We haven't changed any of the data in `memory-datastore` yet, so none 
+of the hints we add to the system will trigger a view refresh to be 
+sent. This is a good thing!
+
+Normally in your application you'll want to add hints to the view 
+system at the same place you do some operation that changes data. So, 
+we can add a function to allow us to change the data in 
+`memory-datastore` and add an appropriate hint about what was changed 
+to the view system at the same time:
+
+```clj
+(defn memdb-assoc-in!
+  [vs namespace ks v]
+  (let [path  (into [namespace] ks)
+        hints [(views/hint namespace ks memory-view-hint-type)]]
+    (swap! memory-datastore assoc-in path v)
+    (views/refresh-views! vs hints)))
+```
+
+And then we can use it to change data relevant to the view we're 
+subscribed to (`:foo`):
+
+```clj
+(memdb-assoc-in! view-system :a [:foo] 42)
+```
+
+As soon as you run this you should see that `send-fn` was called to 
+send out a view refresh:
+
+```
+view refresh 123 {:view-id :foo, :parameters []} 42
+```
+
+And of course, `memory-datastore` was updated correctly at the same 
+time:
+
+```clj
+@memory-datastore
+=> {:a {:foo 42, :bar 200, :baz [1 2 3]}, :b {:foo 2, :bar 300, :baz [2 3 4]}}
+```
+
+As we would expect given the current subscriptions in our view system, 
+view refreshes will only be sent out if we change the data under 
+`[:a :foo]` as refreshes are only processed if there are subscribers 
+for a view.
+
+### Unsubscribing
+
+Unsubscribing a subscriber is done through `views.core/unsubscribe!` 
+and the arguments are the same:
+
+```clj
+(views/unsubscribe! 
+  view-system                        ; view system atom
+  (views/->view-sig :a :foo [])      ; view sig of the view to unsubscribe from
+  123                                ; subscriber key
+  nil)                               ; context
+```
+
+Remember that subscriptions are keyed by view sig, so to unsubscribe 
+from a view, you must use the exact same namespace and parameters that 
+was used to subscribe to it in the first place.
+
+If you need to unsubscribe from all of a subscriber's current 
+subscriptions, you can use `views.core/unsubscribe-all!` which 
+essentially completely removes a subscriber from the views system.
+
+```clj
+(views/unsubscribe-all! view-system 123)   ; where '123' is the subscriber key
+```
+
+### Shutting Down the Views System
+
+You can stop the views system by simply calling `views.core/shutdown!`
+
+```clj
+(views/shutdown! view-system)
+```
+
+This function will by default block until the refresh watcher and all 
+refresh worker threads have finished (they are sent interrupt signals 
+when `shutdown!` is called). If for some reason you do not wish to 
+block, you can pass an additional argument to `shutdown!`:
+
+```clj
+(views/shutdown! view-system true)   ; don't block waiting for threads to terminate
+```
+
+## Subscription Authorization
+
+By default, no subscriptions require authorization. If you wish for 
+some or all views to require some kind of authorization, you should 
+provide an `:auth-fn` option to `views.core/init!`.
+
+This is a function of the form:
+
+```clj
+(fn [view-sig subscriber-key context]
+  ; ...
+  )
+```
+
+It should return true if the subscription is authorized. `context` is 
+the exact value that was passed in as the context argument to 
+`subscribe!`. You might wish to pass in a Ring request map or a user 
+profile for example.
+
+If subscription authorization fails, `subscribe!` returns `nil`.
+
+You can also provide the `:on-unauth-fn` option to `views.core/init!` 
+and set it to a function that will be called in the event that 
+subscription authorization failed. This function takes the same 
+arguments as `:auth-fn`. The return value is not used.
+
+Your application may or may not need this depending on how you have 
+things set up (the fact that `subscribe!` returns `nil` if unauthorized
+may be enough for you). It is just provided as an extra convenience.
+
+## Namespaces
+
+As has been mentioned already, namespaces can be used to isolate 
+subscriptions to views and view refreshes. Typical use of namespaces 
+within a views system would be to set them to something that specifies 
+which database to retrieve view data from when you have multiple 
+databases all with an identical structure.
+
+Namespace information is not included in the actual view refresh data 
+that gets sent to subscribers. It is just considered to be a 
+server-side concern.
+
+Depending on your application, you may be perfectly ok with just 
+passing in the specific namespace needed when creating view 
+subscriptions. However, you can also specify a `:namespace-fn` option 
+in your call to `views.core/init!` and provide a function that will 
+return the namespace to use for all calls to `subscribe!` and 
+`unsubscribe!` that get passed a view sig which **does not** include a 
+namespace in it.
+
+The `:namespace-fn` function should be of the form:
+
+```clj
+(fn [view-sig subscriber-key context]
+  ; ...
+  )
+```
+
+`context` will be whatever was passed in as the context argument to 
+`subscribe!`/`unsubscribe!`.
+
+It bears repeating that `:namespace-fn` will **not** be called even if
+it was set if you use a view sig that includes a `:namespace` key.
+
+For this reason the helper function `->view-sig` includes an extra 
+overload that does not set a namespace.
+
+```clj
+; a view sig that will result in namespace-fn being called (if one is set)
+(views/->view-sig :foo [])
+=> {:view-id :foo, :parameters []}
+
+; a view sig that will always use :a as the namespace, even if a namespace-fn is set
+(views/->view-sig :a :foo [])
+=> {:namespace :a, :view-id :foo, :parameters []}
+```
+
+## View System Initialization Options
+
+There are a number of options that can be provided to 
+`views.core/init!`. The only one that absolutely must be provided for a
+working system is `:send-fn` while all the other default options will
+generally suffice for a non-distributed relatively low-load
+application.
+
+The default options are defined in `views.core/default-options`.
+
+#### `:send-fn`
+
+A function that is used to send view refresh data to subscribers.
+
+```clj
+(fn [subscriber-key [view-sig view-data]]
+  ; ...
+  )
+```
+
+#### `:views`
+
+A list of `IView` instances. These are the views that can be 
+subscribed to. Views can also be added/replaced in the system after 
+initialization by calling `views.core/add-views!`.
+
+#### `:put-hints-fn`
+
+A function that typically will be used by the different views plugin 
+libraries providing view implementations (such as 
+[views.sql](https://github.com/gered/views.sql) or
+[views.honeysql](https://github.com/gered/views.honeysql)) to add 
+hints to the view system. 
+
+This function is used as a common configurable way for these different
+plugin libraries to add hints because the application can provide an
+alternate implementation to e.g. send hints out over a
+distributed messaging service and it will affect all views in the
+system (which would not be possible if all or just some were hard-coded
+to use `queue-hints!` or `refresh-views!`).
+
+```clj
+(fn [^Atom view-system hints]
+  ; ...
+  )
+```
+
+The default implementation is:
+
+```clj
+(fn [^Atom view-system hints]
+  (refresh-views! view-system hints))
+```
+
+#### `:refresh-queue-size`
+
+The size of the internal refresh request queue used to hold refresh
+requests for the refresh worker threads. If you notice some refresh 
+requests being dropped, you may wish to increase this (after of course
+seeing if you have some slow views that could be improved).
+
+Default is `1000`.
+
+#### `:refresh-interval`
+
+An interval in milliseconds at which the refresh watcher thread will 
+check for queued up hints and dispatch relevant view refresh requests
+to the refresh worker threads.
+
+Default is `1000`.
+
+#### `:worker-threads`
+
+The number of refresh worker threads that continually poll for refresh
+requests and handle sending view refreshes to subscribers.
+
+Default is `8`.
+
+#### `:auth-fn`
+
+A function that authorizes view subscriptions. It should return true
+if the subscription is authorized. If this function is not set, no view
+subscriptions will require authorization.
+
+```clj
+(fn [view-sig subscriber-key context]
+  ; ...
+  )
+```
+
+#### `:on-unauth-fn`
+
+A function that is called when subscription authorization fails. The
+return value of this function is not used.
+
+```clj
+(fn [view-sig subscriber-key context]
+  ; ...
+  )
+```
+
+#### `:namespace-fn`
+
+A function that is used during subscription and unsubscription **only**
+if no namespace is specified in the view sig passed in. This function
+should return the namespace to be used for the 
+subscription/unsubscription.
+
+```clj
+(fn [view-sig subscriber-key context]
+  ; ...
+  )
+```
+
+#### `:stats-log-interval`
+
+Interval in milliseconds at which a logger will output an INFO log
+entry with some view system statistics (refreshes/sec, 
+dropped-refreshes/sec, duplicate-refreshes/sec). If not set, no
+logging is done.
+
+
+## Considerations for Distributed Systems
+
+If you're looking to use a views system with an application that will 
+be running on multiple servers, all you really need to do to get the 
+views system working consistently across all the nodes is to make sure
+that when new hints are to be added to the views system, they are sent
+to all application nodes.
+
+For example, you can set up a messaging service (such as RabbitMQ, etc)
+and when you need to add hints to the views system, instead of calling
+`queue-hints!` or `refresh-views!` with the new hints, you simply send
+them to the messaging service. 
+
+Most of the views plugin libraries providing view implementations 
+(such as views.sql) will call `views.core/put-hints!` to add hints to
+the system. `put-hints!` uses whatever the `:put-hints-fn` function was
+set to in the options passed to `views.core/init!`. The default 
+`:put-hints-fn` implementation simply calls `refresh-views!`, but you
+can easily provide an alternative function that sends the hints to a
+messaging service.
+
+Then your application nodes need to listen for hints being received
+from the messaging service. You should then call `queue-hints!` or
+`refresh-views!` with the hints received this way.
+
+
+
 ## License
 
-Copyright © 2014 DiligenceEngine
+Copyright © 2015-2016 Kira Inc.
 
-Authors Dave Della Costa (https://github.com/ddellacosta) and Alexander Hudek (https://github.com/akhudek)
+Original authors:  
+* Dave Della Costa (https://github.com/ddellacosta)  
+* Alexander Hudek (https://github.com/akhudek)
 
-Distributed under the Eclipse Public License either version 1.0 or (at
+Various updates and other changes in this fork by 
-your option) any later version.
+Gered King (https://github.com/gered)
+
+Distributed under the MIT License.

doc/intro.md

@@ -1,3 +0,0 @@
-# Introduction to views
-
-TODO: write [great documentation](http://jacobian.org/writing/great-documentation/what-to-write/)

project.clj

@@ -1,31 +1,29 @@
-(defproject views "0.4.1"
+(defproject net.gered/views "1.7.0-SNAPSHOT"
-  :description "You underestimate the power of the SQL side"
+  :description         "A view to the past helps navigate the future."
+  :url                 "https://github.com/gered/views"
+  :license             {:name "MIT License"
+                        :url "http://opensource.org/licenses/MIT"}
 
-  :url "https://github.com/diligenceengine/views"
+  :dependencies        [[org.clojure/tools.logging "1.2.4"]]
 
-  :license {:name "Eclipse Public License"
+  :profiles            {:provided
-            :url "http://www.eclipse.org/legal/epl-v10.html"}
+                        {:dependencies [[org.clojure/clojure "1.10.3"]]}
 
-  :dependencies [[org.clojure/clojure "1.6.0"]
+                        :test
-                 [org.clojure/tools.logging "0.2.6"]
+                        {:dependencies [[pjstadig/humane-test-output "0.11.0"]]
-                 [org.clojure/core.async "0.1.303.0-886421-alpha"]
+                         :injections   [(require 'pjstadig.humane-test-output)
-                 [org.clojure/java.jdbc "0.3.3"]
+                                        (pjstadig.humane-test-output/activate!)]}}
-                 [honeysql "0.4.3"]
-                 [edl "0.1.0"]
-                 [org.postgresql/postgresql "9.2-1003-jdbc4"]
-                 [clj-logging-config "1.9.10"]
-                 [zip-visit "1.0.2"]
-                 [pjstadig/humane-test-output "0.6.0"]
 
-                 ;; Metrics
+  :deploy-repositories [["releases" :clojars]
-                 [riemann-clojure-client "0.2.11"]
+                        ["snapshots" :clojars]]
-                 ]
 
-  :profiles {:test {:dependencies [[org.clojure/tools.nrepl "0.2.3"]
+  :release-tasks       [["vcs" "assert-committed"]
-                                   [environ "0.4.0"]
+                        ["change" "version" "leiningen.release/bump-version" "release"]
-                                   [org.clojure/data.generators "0.1.2"]]
+                        ["vcs" "commit"]
+                        ["vcs" "tag" "v" "--no-sign"]
+                        ["deploy"]
+                        ["change" "version" "leiningen.release/bump-version"]
+                        ["vcs" "commit" "bump to next snapshot version for future development"]
+                        ["vcs" "push"]]
 
-                    :injections [(require 'pjstadig.humane-test-output)
+  )
-                                 (pjstadig.humane-test-output/activate!)]}}
-
-  :plugins [[lein-environ "0.4.0"]])

src/views/base_subscribed_views.clj

@@ -1,155 +0,0 @@
-(ns views.base-subscribed-views
-  (:require
-   [views.persistence.core :as persist]
-   [views.subscribed-views :refer [ISubscribedViews]]
-   [views.filters :refer [view-filter]]
-   [views.db.load :refer [initial-view]]
-   [views.db.util :refer [with-retry]]
-   [clojure.tools.logging :refer [debug info warn error]]
-   [clojure.core.async :refer [put! <! go thread]]
-   [clojure.java.jdbc :as j]
-
-   ;; Metrics
-   [views.riemann :refer [rclient]]
-   [riemann.client :refer [send-event]]
-   ))
-
-(def default-ns :default-ns)
-
-(declare send-deltas)
-
-(defn send-fn*
-  [send-fn address subject msg]
-  (if send-fn
-    (send-fn address subject msg)
-    (warn "IMPLEMENT ME. Got message " msg " with subject " subject " sent to address " address)))
-
-(defn subscriber-key-fn*
-  [subscriber-key-fn msg]
-  (if subscriber-key-fn (subscriber-key-fn msg) (:subscriber-key msg)))
-
-(defn namespace-fn*
-  [namespace-fn msg]
-  (if namespace-fn (namespace-fn msg) default-ns))
-
-(defn view-sig-fn*
-  [view-sig-fn msg]
-  (if view-sig-fn (view-sig-fn msg) (:body msg)))
-
-(defn subscribe-and-compute
-  "Subscribe a view and return the initial values."
-  [db persistence templates vs namespace subscriber-key]
-  (let [view-data (persist/subscribe! persistence templates namespace vs subscriber-key)]
-    (with-retry
-      (j/with-db-transaction [t db :isolation :serializable]
-          (initial-view t vs templates (:view view-data))))))
-
-;; Deltas look like:
-;; [{view-sig1 delta, view-sig2 delta, ...} {view-sig3 delta, ...}]
-
-(defn delta-signatures
-  "Return all the signatures mentioned by a map of deltas."
-  [deltas]
-  (mapcat keys deltas))
-
-(deftype BaseSubscribedViews [config]
-  ISubscribedViews
-  (subscribe-views
-    [this msg]
-    (let [{:keys [persistence templates db-fn send-fn view-sig-fn subscriber-key-fn namespace-fn unsafe?]} config
-          db             (if db-fn (db-fn msg) (:db config))
-          subscriber-key (subscriber-key-fn* subscriber-key-fn msg)
-          namespace      (namespace-fn* namespace-fn msg)
-          view-sigs      (view-filter msg (view-sig-fn* view-sig-fn msg) templates {:unsafe? unsafe?})] ; this is where security comes in.
-      (debug "Subscribing views: " view-sigs " for subscriber " subscriber-key ", in namespace " namespace)
-      (when (seq view-sigs)
-          (doseq [vs view-sigs]
-            (thread
-              (let [iv (subscribe-and-compute db persistence templates vs namespace subscriber-key)
-                    start  (System/currentTimeMillis)]
-                (send-fn* send-fn subscriber-key :views.init iv)
-                (send-event rclient {:service "subscription-init-time" :metric (- (System/currentTimeMillis) start)})))))))
-
-  (unsubscribe-views
-    [this msg]
-    (let [{:keys [subscriber-key-fn namespace-fn persistence view-sig-fn]} config
-          subscriber-key (subscriber-key-fn* subscriber-key-fn msg)
-          namespace      (namespace-fn* namespace-fn msg)
-          view-sigs      (view-sig-fn* view-sig-fn msg)]
-      (debug "Unsubscribing views: " view-sigs " for subscriber " subscriber-key)
-      (doseq [vs view-sigs]
-        (persist/unsubscribe! persistence namespace vs subscriber-key))))
-
-  (disconnect [this msg]
-    (let [{:keys [subscriber-key-fn namespace-fn persistence]} config
-          subscriber-key (subscriber-key-fn* subscriber-key-fn msg)
-          namespace      (namespace-fn* namespace-fn msg)]
-      (debug "Disconnecting subscriber " subscriber-key " in namespace " namespace)
-      (persist/unsubscribe-all! persistence namespace subscriber-key)))
-
-  ;;
-  ;; The two below functions get called by vexec!/with-view-transaction
-  ;;
-
-  (subscribed-views [this namespace]
-    ;; Table name optimization not yet worked through the library.
-    (persist/view-data (:persistence config) namespace "fix-me"))
-
-  (broadcast-deltas [this deltas namespace]
-    (let [{:keys [templates]} config
-          namespace (if namespace namespace default-ns)
-          subs      (persist/subscriptions (:persistence config) namespace (delta-signatures deltas))]
-      (send-deltas deltas subs namespace config))))
-
-(defn post-process-delta-map
-  [post-fn delta-map]
-  (if-let [rset (:refresh-set delta-map)]
-    delta-map
-    (reduce #(assoc %1 %2 (map post-fn (get delta-map %2))) {} (keys delta-map))))
-
-(defn post-process-deltas
-  "Run post-processing functions on each delta. NOTE: this puts things in maps
-  to maintain compatability with the frontend code."
-  [delta templates]
-  (let [vs (first delta)]
-    (if-let [post-fn (get-in templates [(first vs) :post-fn])]
-      {(first delta) (mapv #(post-process-delta-map post-fn %) (second delta))}
-      {(first delta) (second delta)})))
-
-;; We flatten the above into a sequence:
-;; [[view-sig1 delta-data], [view-sig2 delta-data]....]
-;; where the signatures from each pack are listed in order.
-
-(defn flatten-deltas
-  "We flatten the above into a sequence:
-   [[view-sig1 delta-data], [view-sig2 delta-data]....]
-   where the signatures from each pack are listed in order."
-  [deltas]
-  (reduce #(into %1 (seq %2)) [] deltas))
-
-(defn update-subscriber-pack
-  "Given a delta [view-sig delta-data] we find the subscribers that need it
-  and add to the subscriber pack vector {view-sig [delta...]}."
-  [subs spacks delta]
-  (let [subscribers (get subs (ffirst delta))]
-    (reduce #(update-in %1 [%2] (fnil conj []) delta) spacks subscribers)))
-
-(defn subscriber-deltas
-  "Group deltas into subscriber packs."
-  [subs deltas]
-  (reduce #(update-subscriber-pack subs %1 %2) {} deltas))
-
-;; Deltas looks like:
-;; [delta-pack1 delta-pack2 ...]
-;; where each delta pack is a map:
-;; {view-sig1 delta-data, view-sig2 delta-data, ...}
-
-(defn send-deltas
-  "Send deltas out to subscribers."
-  [deltas subs namespace {:keys [send-fn templates] :as config}]
-  (let [deltas (mapv #(post-process-deltas % templates) (flatten-deltas deltas))
-        start  (System/currentTimeMillis)]
-    (doseq [[sk deltas*] (subscriber-deltas subs deltas)]
-      (debug "Sending deltas " deltas* " to subscriber " sk)
-      (send-fn* send-fn sk :views.deltas deltas*))
-    (send-event rclient {:service "delta-send-time" :metric (- (System/currentTimeMillis) start)})))

src/views/core.clj

@@ -1,18 +1,509 @@
 (ns views.core
-  (:require
-   [views.base-subscribed-views :as bsv]
-   [views.core :as vp]
-   [edl.schema :refer [denormalized-schema get-schema]]
-   [views.persistence.memory :refer [new-memory-persistence]])
   (:import
-   [views.base_subscribed_views BaseSubscribedViews]))
+    (java.util.concurrent ArrayBlockingQueue TimeUnit)
+    (clojure.lang Atom))
+  (:require
+    [views.protocols :refer [IView id data relevant?]]
+    [clojure.tools.logging :refer [info debug error trace]]))
 
-(defn config
+;; The view-system data structure has this shape:
-  [{:keys [db templates persistence vexec-ns-fn] :as conf}]
+;;
-  (let [schema (denormalized-schema (get-schema db (get conf :schema-name "public")))
+;; {
-        conf   (if persistence conf (assoc conf :persistence (new-memory-persistence)))]
+;;
-    {:db                    db
+;;  :refresh-queue (ArrayBlockingQueue.)
-     :schema                schema
+;;  :views         {:id1 view1, id2 view2, ...}
-     :templates             templates
+;;  :send-fn       (fn [subscriber-key data] ...)
-     :vexec-ns-fn           vexec-ns-fn
+;;  :put-hints-fn  (fn [hints] ... )
-     :base-subscribed-views (BaseSubscribedViews. conf)}))
+;;  :auth-fn       (fn [view-sig subscriber-key context] ...)
+;;  :namespace-fn  (fn [view-sig subscriber-key context] ...)
+;;
+;;  :hashes        {view-sig hash, ...}
+;;  :subscribed    {subscriber-key #{view-sig, ...}}
+;;  :subscribers   {view-sig #{subscriber-key, ...}}
+;;  :hints         #{hint1 hint2 ...}
+;;
+;;  }
+;;
+;;  Each hint has the form {:namespace x :hint y}
+
+
+
+(defn reset-stats!
+  "Resets statistics collected back to zero."
+  [^Atom view-system]
+  (swap! view-system update-in [:statistics] assoc
+         :refreshes 0
+         :dropped 0
+         :deduplicated 0)
+  view-system)
+
+(defn collecting-stats?
+  "Whether view statem statistics collection and logging is enabled or not."
+  [^Atom view-system]
+  (boolean (get-in @view-system [:statistics :logger])))
+
+(defn ->view-sig
+  ([namespace view-id parameters]
+   {:namespace  namespace
+    :view-id    view-id
+    :parameters parameters})
+  ([view-id parameters]
+   {:view-id    view-id
+    :parameters parameters}))
+
+(defn- send-view-data!
+  [view-system subscriber-key {:keys [namespace view-id parameters] :as view-sig} data]
+  (if-let [send-fn (:send-fn view-system)]
+    (send-fn subscriber-key [(dissoc view-sig :namespace) data])
+    (throw (new Exception "no send-fn function set in view-system"))))
+
+(defn- authorized-subscription?
+  [view-system view-sig subscriber-key context]
+  (if-let [auth-fn (:auth-fn view-system)]
+    (auth-fn view-sig subscriber-key context)
+    ; assume that if no auth-fn is specified, that we are not doing auth checks at all
+    ; so do not disallow access to any subscription
+    true))
+
+(defn- on-unauthorized-subscription
+  [view-system view-sig subscriber-key context]
+  (if-let [on-unauth-fn (:on-unauth-fn view-system)]
+    (on-unauth-fn view-sig subscriber-key context)))
+
+(defn- get-namespace
+  [view-system view-sig subscriber-key context]
+  (if-let [namespace-fn (:namespace-fn view-system)]
+    (namespace-fn view-sig subscriber-key context)
+    (:namespace view-sig)))
+
+(defn- subscribe-view!
+  [view-system view-sig subscriber-key]
+  (trace "subscribing to view" view-sig subscriber-key)
+  (-> view-system
+      (update-in [:subscribed subscriber-key] (fnil conj #{}) view-sig)
+      (update-in [:subscribers view-sig] (fnil conj #{}) subscriber-key)))
+
+(defn- update-hash!
+  [view-system view-sig data-hash]
+  (update-in view-system [:hashes view-sig] #(or % data-hash))) ;; see note #1 in NOTES.md
+
+(defn subscribe!
+  "Creates a subscription to a view identified by view-sig for a subscriber
+   identified by subscriber-key. If the subscription is not authorized,
+   returns nil. Additional context info can be passed in, which will be
+   passed to the view-system's namespace-fn and auth-fn (if provided). If
+   the subscription is successful, the subscriber will be sent the initial
+   data for the view."
+  [^Atom view-system {:keys [namespace view-id parameters] :as view-sig} subscriber-key context]
+  (if-let [view (get-in @view-system [:views view-id])]
+    (let [namespace (if (contains? view-sig :namespace)
+                      namespace
+                      (get-namespace @view-system view-sig subscriber-key context))
+          view-sig  (->view-sig namespace view-id parameters)]
+      (if (authorized-subscription? @view-system view-sig subscriber-key context)
+        (do
+          (swap! view-system subscribe-view! view-sig subscriber-key)
+          (future
+            (try
+              (let [vdata     (data view namespace parameters)
+                    data-hash (hash vdata)]
+                ;; Check to make sure that we are still subscribed. It's possible that
+                ;; an unsubscription event came in while computing the view.
+                (when (contains? (get-in @view-system [:subscribed subscriber-key]) view-sig)
+                  (swap! view-system update-hash! view-sig data-hash)
+                  (send-view-data! @view-system subscriber-key view-sig vdata)))
+              (catch Exception e
+                (error e "error subscribing to view" view-sig)))))
+        (do
+          (trace "subscription not authorized" view-sig subscriber-key context)
+          (on-unauthorized-subscription @view-system view-sig subscriber-key context)
+          nil)))
+    (throw (new Exception (str "Subscription for non-existant view: " view-id)))))
+
+(defn- remove-from-subscribers
+  [view-system view-sig subscriber-key]
+  (-> view-system
+      (update-in [:subscribers view-sig] disj subscriber-key)
+      ; remove view-sig entry if no subscribers. helps prevent the subscribers
+      ; map from e.g. endlessly filling up with all sorts of different
+      ; view-sigs with crazy amounts of only-slightly-varying parameters
+      (update-in [:subscribers]
+                 (fn [subscribers]
+                   (if (empty? (get subscribers view-sig))
+                     (dissoc subscribers view-sig)
+                     subscribers)))))
+
+(defn- remove-from-subscribed
+  [view-system view-sig subscriber-key]
+  (-> view-system
+      (update-in [:subscribed subscriber-key] disj view-sig)
+      ; remove subscriber-key entry if no current subscriptions. this helps prevent
+      ; the subscribed map from (for example) endlessly filling up with massive
+      ; amounts of entries with no subscriptions. this could easily happen over time
+      ; naturally for applications with long uptimes.
+      (update-in [:subscribed]
+                 (fn [subscribed]
+                   (if (empty? (get subscribed subscriber-key))
+                     (dissoc subscribed subscriber-key)
+                     subscribed)))))
+
+(defn- clean-up-unneeded-hashes
+  [view-system view-sig]
+  ; hashes for view-sigs which do not have any unsubscribers are no longer necessary
+  ; to keep around (again, at risk of endlessly filling up with tons of hashes over time)
+  (if-not (get (:subscribers view-system) view-sig)
+    (update-in view-system [:hashes] dissoc view-sig)
+    view-system))
+
+(defn unsubscribe!
+  "Removes a subscription to a view identified by view-sig for a subscriber
+   identified by subscriber-key. Additional context info can be passed in,
+   which will be passed to the view-system's namespace-fn (if provided)."
+  [^Atom view-system {:keys [namespace view-id parameters] :as view-sig} subscriber-key context]
+  (trace "unsubscribing from view" view-sig subscriber-key)
+  (swap! view-system
+         (fn [view-system]
+           (let [namespace (if (contains? view-sig :namespace)
+                             namespace
+                             (get-namespace view-system view-sig subscriber-key context))
+                 view-sig  (->view-sig namespace view-id parameters)]
+             (-> view-system
+                 (remove-from-subscribed view-sig subscriber-key)
+                 (remove-from-subscribers view-sig subscriber-key)
+                 (clean-up-unneeded-hashes view-sig)))))
+  view-system)
+
+(defn unsubscribe-all!
+  "Removes all of a subscriber's (identified by subscriber-key) current
+   view subscriptions."
+  [^Atom view-system subscriber-key]
+  (trace "unsubscribing from all views" subscriber-key)
+  (swap! view-system
+         (fn [view-system]
+           (let [view-sigs    (get-in view-system [:subscribed subscriber-key])
+                 view-system* (update-in view-system [:subscribed] dissoc subscriber-key)]
+             (reduce
+               #(-> %1
+                    (remove-from-subscribers %2 subscriber-key)
+                    (clean-up-unneeded-hashes %2))
+               view-system*
+               view-sigs))))
+  view-system)
+
+(defn refresh-view!
+  "Schedules a view (identified by view-sig) to be refreshed by one of the worker threads
+   only if the provided collection of hints is relevant to that view."
+  [^Atom view-system hints {:keys [namespace view-id parameters] :as view-sig}]
+  (let [v (get-in @view-system [:views view-id])]
+    (if-let [^ArrayBlockingQueue refresh-queue (:refresh-queue @view-system)]
+      (try
+        (if (relevant? v namespace parameters hints)
+          (if-not (.contains refresh-queue view-sig)
+            (when-not (.offer refresh-queue view-sig)
+              (if (collecting-stats? view-system) (swap! view-system update-in [:statistics :dropped] inc))
+              (error "refresh-queue full, dropping refresh request for" view-sig))
+            (do
+              (if (collecting-stats? view-system) (swap! view-system update-in [:statistics :deduplicated] inc))
+              (trace "already queued for refresh" view-sig))))
+        (catch Exception e
+          (error e "error determining if view is relevant" view-sig))))
+    view-system))
+
+(defn subscribed-views
+  "Returns a list of all views in the system that have subscribers."
+  [^Atom view-system]
+  (reduce into #{} (vals (:subscribed @view-system))))
+
+(defn active-view-count
+  "Returns a count of views with at least one subscriber."
+  [^Atom view-system]
+  (count (remove #(empty? (val %)) (:subscribers @view-system))))
+
+(defn- pop-hints!
+  [^Atom view-system]
+  (let [p (swap-vals! view-system assoc :hints #{})]
+    (or (:hints (first p)) #{})))
+
+(defn refresh-views!
+  "Given a collection of hints, check all views in the system to find any that need refreshing
+   and schedule refreshes for them. If no hints are provided, will use any that have been
+   queued up in the view-system."
+  ([^Atom view-system hints]
+   (when (seq hints)
+     (trace "refresh hints:" hints)
+     (doseq [view-sig (subscribed-views view-system)]
+       (refresh-view! view-system hints view-sig)))
+   (swap! view-system assoc :last-update (System/currentTimeMillis))
+   view-system)
+  ([^Atom view-system]
+   (refresh-views! view-system (pop-hints! view-system))))
+
+(defn- can-refresh?
+  [last-update min-refresh-interval]
+  (> (- (System/currentTimeMillis) last-update) min-refresh-interval))
+
+(defn- wait
+  [last-update min-refresh-interval]
+  (Thread/sleep (max 0 (- min-refresh-interval (- (System/currentTimeMillis) last-update)))))
+
+(defn do-view-refresh!
+  [^Atom view-system {:keys [namespace view-id parameters] :as view-sig}]
+  (if (collecting-stats? view-system) (swap! view-system update-in [:statistics :refreshes] inc))
+  (try
+    (let [view  (get-in @view-system [:views view-id])
+          vdata (data view namespace parameters)
+          hdata (hash vdata)]
+      (when-not (= hdata (get-in @view-system [:hashes view-sig]))
+        (doseq [subscriber-key (get-in @view-system [:subscribers view-sig])]
+          (send-view-data! @view-system subscriber-key view-sig vdata))
+        (swap! view-system assoc-in [:hashes view-sig] hdata)))
+    (catch Exception e
+      (error e "error refreshing:" namespace view-id parameters))))
+
+(defn- refresh-worker-thread
+  [^Atom view-system]
+  (let [^ArrayBlockingQueue refresh-queue (:refresh-queue @view-system)]
+    (fn []
+      (try
+        (when-let [view-sig (.poll refresh-queue 60 TimeUnit/SECONDS)]
+          (trace "worker running refresh for" view-sig)
+          (do-view-refresh! view-system view-sig))
+        (catch InterruptedException e))
+      (if-not (:stop-workers? @view-system)
+        (recur)
+        (trace "exiting worker thread")))))
+
+(defn- refresh-watcher-thread
+  [^Atom view-system min-refresh-interval]
+  (fn []
+    (let [last-update (:last-update @view-system)]
+      (try
+        (if (can-refresh? last-update min-refresh-interval)
+          (refresh-views! view-system)
+          (wait last-update min-refresh-interval))
+        (catch InterruptedException e)
+        (catch Exception e
+          (error e "exception in views")))
+      (if-not (:stop-refresh-watcher? @view-system)
+        (recur)
+        (trace "exiting refresh watcher thread")))))
+
+(defn start-update-watcher!
+  "Starts threads for the views refresh watcher and worker threads that handle queued
+   hints and view refresh requests."
+  [^Atom view-system min-refresh-interval threads]
+  (trace "starting refresh watcher at" min-refresh-interval "ms interval and" threads "workers")
+  (if (and (:refresh-watcher @view-system)
+           (:workers @view-system))
+    (error "cannot start new watcher and worker threads until existing threads are stopped")
+    (let [refresh-watcher (Thread. ^Runnable (refresh-watcher-thread view-system min-refresh-interval))
+          worker-threads  (mapv (fn [_] (Thread. ^Runnable (refresh-worker-thread view-system)))
+                                (range threads))]
+      (swap! view-system assoc
+             :last-update 0
+             :refresh-watcher refresh-watcher
+             :stop-refresh-watcher? false
+             :workers worker-threads
+             :stop-workers? false)
+      (.start refresh-watcher)
+      (doseq [^Thread t worker-threads]
+        (.start t))
+      view-system)))
+
+(defn stop-update-watcher!
+  "Stops threads for the views refresh watcher and worker threads."
+  [^Atom view-system & [dont-wait-for-threads?]]
+  (trace "stopping refresh watcher and workers")
+  (let [worker-threads (:workers @view-system)
+        watcher-thread (:refresh-watcher @view-system)
+        threads        (->> worker-threads
+                            (cons watcher-thread)
+                            (remove nil?))]
+    (swap! view-system assoc
+           :stop-refresh-watcher? true
+           :stop-workers? true)
+    (doseq [^Thread t threads]
+      (.interrupt t))
+    (if-not dont-wait-for-threads?
+      (doseq [^Thread t threads]
+        (.join t)))
+    (swap! view-system assoc
+           :refresh-watcher nil
+           :workers nil))
+  view-system)
+
+(defn- logger-thread
+  [^Atom view-system msecs]
+  (let [secs (/ msecs 1000)]
+    (fn []
+      (try
+        (Thread/sleep msecs)
+        (let [stats (:statistics @view-system)]
+          (reset-stats! view-system)
+          (info "subscribed views:" (active-view-count view-system)
+                (format "refreshes/sec: %.1f" (double (/ (:refreshes stats) secs)))
+                (format "dropped/sec: %.1f" (double (/ (:dropped stats) secs)))
+                (format "deduped/sec: %.1f" (double (/ (:deduplicated stats) secs)))))
+        (catch InterruptedException e))
+      (if-not (get-in @view-system [:statistics :stop?])
+        (recur)))))
+
+(defn start-logger!
+  "Starts a logger thread that will enable collection of view statistics
+   which the logger will periodically write out to the log."
+  [^Atom view-system log-interval]
+  (trace "starting logger. logging at" log-interval "secs intervals")
+  (if (get-in @view-system [:statistics :logger])
+    (error "cannot start new logger thread until existing thread is stopped")
+    (let [logger (Thread. ^Runnable (logger-thread view-system log-interval))]
+      (swap! view-system update-in [:statistics] assoc
+             :logger logger
+             :stop? false)
+      (reset-stats! view-system)
+      (.start logger)))
+  view-system)
+
+(defn stop-logger!
+  "Stops the logger thread."
+  [^Atom view-system & [dont-wait-for-thread?]]
+  (trace "stopping logger")
+  (let [^Thread logger-thread (get-in @view-system [:statistics :logger])]
+    (swap! view-system assoc-in [:statistics :stop?] true)
+    (if logger-thread (.interrupt logger-thread))
+    (if-not dont-wait-for-thread? (.join logger-thread))
+    (swap! view-system assoc-in [:statistics :logger] nil))
+  view-system)
+
+(defn hint
+  "Create a hint."
+  [namespace hint type]
+  {:namespace namespace :hint hint :type type})
+
+(defn queue-hints!
+  "Queues up hints in the view system so that they will be picked up by the refresh
+   watcher and dispatched to the workers resulting in view updates being sent out
+   for the relevant views/subscribers."
+  [^Atom view-system hints]
+  (trace "queueing hints" hints)
+  (swap! view-system update-in [:hints] (fnil into #{}) hints)
+  view-system)
+
+(defn put-hints!
+  "Adds a collection of hints to the view system by using the view system
+   configuration's :put-hints-fn."
+  [^Atom view-system hints]
+  ((:put-hints-fn @view-system) view-system hints)
+  view-system)
+
+(defn- ->views-map
+  [views]
+  (map vector (map id views) views))
+
+(defn add-views!
+  "Add a collection of views to the system."
+  [^Atom view-system views]
+  (swap! view-system update-in [:views] (fnil into {}) (->views-map views))
+  view-system)
+
+(def default-options
+  "Default options used to initialize the views system via init!"
+  {
+   ; *REQUIRED*
+   ; a function that is used to send view refresh data to subscribers.
+   ; this function must be set for normal operation of the views system.
+   ; (fn [subscriber-key [view-sig view-data]] ...)
+   :send-fn            nil
+
+   ; *REQUIRED*
+   ; a function that adds hints to the view system. this function will be used
+   ; by other libraries that implement IView. this function must be set for
+   ; normal operation of the views system. the default function provided
+   ; will trigger relevant view refreshes immediately.
+   ; (fn [^Atom view-system hints] ... )
+   :put-hints-fn       (fn [^Atom view-system hints] (refresh-views! view-system hints))
+
+   ; *REQUIRED*
+   ; the size of the queue used to hold view refresh requests for
+   ; the worker threads. for very heavy systems, this can be set
+   ; higher if you start to get warnings about dropped refresh requests
+   :refresh-queue-size 1000
+
+   ; *REQUIRED*
+   ; interval in milliseconds at which the refresh watcher thread will
+   ; check for any queued up hints and dispatch relevant view refresh
+   ; updates to the worker threads.
+   :refresh-interval   1000
+
+   ; *REQUIRED*
+   ; the number of refresh worker threads that poll for view refresh
+   ; requests and dispatch updated view data to subscribers.
+   :worker-threads     8
+
+   ; a list of IView instances. these are the views that can be subscribed
+   ; to. views can also be added/replaced after system initialization through
+   ; the use of add-views!
+   :views              nil
+
+   ; a function that authorizes view subscriptions. should return true if the
+   ; subscription is authorized. if not set, no view subscriptions will require
+   ; any authorization.
+   ; (fn [view-sig subscriber-key context] ... )
+   :auth-fn            nil
+
+   ; a function that is called when subscription authorization fails.
+   ; (fn [view-sig subscriber-key context] ... )
+   :on-unauth-fn       nil
+
+   ; a function that returns a namespace to use for view subscriptions
+   ; (fn [view-sig subscriber-key context] ... )
+   :namespace-fn       nil
+
+   ; interval in milliseconds at which a logger will write view system
+   ; statistics to the log. if not set, the logger will be disabled.
+   :stats-log-interval nil
+   })
+
+(defn init!
+  "Initializes the view system for use with the list of views provided.
+
+   An existing atom that will be used to store the state of the views
+   system can be provided, otherwise one will be created. Either way,
+   the atom with the initialized view system is returned.
+
+   options is a map of options to configure the view system with. See
+   views.core/default-options for a description of the available options
+   and the defaults that will be used for any options not provided in
+   the call to init!."
+  ([^Atom view-system options]
+   (let [options (merge default-options options)]
+     (trace "initializing views system using options:" options)
+     (reset! view-system
+             {:refresh-queue (ArrayBlockingQueue. (:refresh-queue-size options))
+              :views         (into {} (->views-map (:views options)))
+              :send-fn       (:send-fn options)
+              :put-hints-fn  (:put-hints-fn options)
+              :auth-fn       (:auth-fn options)
+              :on-unauth-fn  (:on-unauth-fn options)
+              :namespace-fn  (:namespace-fn options)
+              ; keeping a copy of the options used during init allows other libraries
+              ; that plugin/extend views functionality (e.g. IView implementations)
+              ; to make use of any options themselves
+              :options       options})
+     (start-update-watcher! view-system (:refresh-interval options) (:worker-threads options))
+     (when-let [stats-log-interval (:stats-log-interval options)]
+       (swap! view-system assoc :logging? true)
+       (start-logger! view-system stats-log-interval))
+     view-system))
+  ([options]
+    (init! (atom {}) options)))
+
+(defn shutdown!
+  "Shuts the view system down, terminating all worker threads and clearing
+   all view subscriptions and data."
+  [^Atom view-system & [dont-wait-for-threads?]]
+  (trace "shutting down views sytem")
+  (stop-update-watcher! view-system dont-wait-for-threads?)
+  (if (:logging? @view-system)
+    (stop-logger! view-system dont-wait-for-threads?))
+  (reset! view-system {})
+  view-system)

src/views/db/checks.clj

@@ -1,46 +0,0 @@
-(ns views.db.checks
-  (:require
-   [views.db.honeysql :as vh]
-   [clojure.set :refer [intersection]]
-   [clojure.zip :as z]
-   [zip.visit :as zv]
-   [honeysql.core :as hsql]))
-
-(defn replace-param-pred
-  []
-  (zv/visitor
-   :pre [n s]
-   (if (and (coll? n) (string? (last n)) (= (subs (last n) 0 1) "?"))
-     {:node true
-      :state (conj s n)})))
-
-(defn swap-wc-preds
-  [wc]
-  (let [root (z/vector-zip wc)]
-    (zv/visit root nil [(replace-param-pred)])))
-
-(defn swap-preds
-  [vm]
-  (let [{:keys [node state]} (swap-wc-preds (:where vm))]
-    {:q (assoc vm :where node) :p state}))
-
-(defn view-sig->dummy-args
-  [view-sig]
-  (map #(str "?" %) (range 0 (count (rest view-sig)))))
-
-(defn view-check
-  [action dummy-vm]
-  (let [{:keys [p q]} (swap-preds dummy-vm)]
-    (-> q
-        (update-in [:where] #(merge % (:where action)))
-        (assoc :select (mapv second p)))))
-
-(defn have-overlapping-tables?
-  "Takes two Honeysql hash-maps, one for action, one for view, and returns
-   boolean value representing whether or not their set of tables intersect."
-  [action view]
-  (->> [action view]
-       (map (comp set #(map first %) vh/extract-tables))
-       (apply intersection)
-       seq
-       boolean))

src/views/db/core.clj

@@ -1,56 +0,0 @@
-(ns views.db.core
-  (:require
-   [clojure.java.jdbc :as j]
-   [clojure.tools.logging :refer [debug]]
-   [views.db.deltas :as vd]
-   [views.db.util :refer [with-retry retry-on-transaction-failure]]
-   [views.subscribed-views :refer [subscribed-views broadcast-deltas]]))
-
-(defmacro with-view-transaction
-  "Like with-db-transaction, but operates with views. If you want to use a
-  standard jdbc function, the transcation database map is accessible with
-  (:db vt) where vt is the bound view transaction."
-  [binding & forms]
-  (let [tvar (first binding), vc (second binding)]
-    `(if (:deltas ~vc) ;; check if we are in a nested transaction
-       (let [~tvar ~vc] ~@forms)
-       (let [base-subscribed-views# (:base-subscribed-views ~vc)
-             deltas#  (atom [])
-             result#  (with-retry
-                        (j/with-db-transaction [t# (:db ~vc) :isolation :serializable]
-                          (let [~tvar (assoc ~vc :deltas deltas# :db t#)]
-                            ~@forms)))]
-         (broadcast-deltas base-subscribed-views# @deltas# (:namespace ~vc))
-         result#))))
-
-(defn vexec!
-  "Used to perform arbitrary insert/update/delete actions on the database,
-   while ensuring that view deltas are appropriately checked and calculated
-   for the currently registered views as reported by a type implementing
-   the ISubscribedViews protocol.
-
-   Arguments are:
-
-   - schema: an edl schema (\"(defschema my-schema ...)\")
-
-   - db: a clojure.java.jdbc database
-
-   - action-map: the HoneySQL map for the insert/update/delete action
-
-   - subscribed-views: an implementation of ISubscribedViews implementing
-                       the follow functions:
-
-     - subscribed-views takes a ... . It should return
-       a collection of view-maps.
-
-     - broadcast-deltas takes ... ."
-  [{:keys [db schema base-subscribed-views templates namespace deltas] :as conf} action-map]
-  (let [subbed-views    (subscribed-views base-subscribed-views namespace)
-        transaction-fn #(vd/do-view-transaction schema db subbed-views action-map templates)]
-    (if deltas  ;; inside a transaction we just collect deltas and do not retry
-      (let [{:keys [new-deltas result-set]} (transaction-fn)]
-        (swap! deltas #(conj % new-deltas))
-        result-set)
-      (let [{:keys [new-deltas result-set]} (retry-on-transaction-failure transaction-fn)]
-        (broadcast-deltas base-subscribed-views [new-deltas] namespace)
-        result-set))))

src/views/db/deltas.clj

@@ -1,263 +0,0 @@
-(ns views.db.deltas
-  (:require
-   [clojure.string :refer [split]]
-   [clojure.java.jdbc :as j]
-   [clojure.tools.logging :refer [debug]]
-   [honeysql.core :as hsql]
-   [honeysql.helpers :as hh]
-   [views.db.load :as vdbl]
-   [views.db.checks :as vc]
-   [views.db.honeysql :as vh]))
-
-;;
-;; Terminology and data structures used throughout this code
-;;
-;; <name>-template - refers to a function which receives parameters
-;;                   and returns a HoneySQL hash-map with params interpolated.
-;;
-;; action          - describes the HoneySQL hash-map for the action to be performed
-;;                   --the template function has already been called and returned this
-;;                   with the appropriate parameter arguments.
-;;
-;; view-map        - contains a set of computed information for each view itself.
-;;                   Refer to the view-map doc-string for more information.
-;;
-
-(defn view-map
-  "Constructs a view map from a HoneySQL view function and its arguments.
-   Contains four fields:
-    :view          - the hash-map with interpolated parameters
-    :view-sig      - the \"signature\" for the view, i.e. [:matter 1]
-    :tables        - the tables present in all :from, :insert-into,
-                 :update, :delete-from, :join, :left-join :right-join clauses
-
-   Input is a view template function and a view signature. The template
-   function must take the same number of paramters as the signature and
-   return a honeysql data structure "
-  [view-template view-sig]
-  (let [compiled-view (if (> (count view-sig) 1)
-                        (apply view-template (rest view-sig))
-                        (view-template))]
-    {:view-sig      view-sig
-     :view          compiled-view
-     :refresh-only? (:refresh-only (meta view-template))}))
-
-(defn view-sig->view-map
-  "Takes a map of sig keys to view template function vars (templates)
-   and a view signature (view-sig the key for the template map and its args)
-   and returns a view-map for that view-sig."
-  [templates view-sig]
-  (let [lookup (first view-sig)]
-    (view-map (get-in templates [lookup :fn]) view-sig)))
-
-;; Helpers
-
-(defn get-primary-key
-  "Get a primary key for a table."
-  [schema table]
-  (or
-    (keyword (get-in schema [(name table) :primary-key :column_name]))
-    (throw (Exception. (str "Cannot find primary key for table: " table)))))
-
-(defn- create-view-delta-where-clauses
-  [view-map action]
-  (let [action-table (first (vh/extract-tables action))
-        view-tables  (vh/extract-tables (:view view-map))]
-    (for [view-table (vh/find-table-aliases action-table view-tables)]
-      (-> (:where action)
-          (vh/prefix-columns (vh/table-alias view-table))
-          (vh/replace-table (vh/table-alias action-table) (vh/table-alias view-table))))))
-
-(defn format-action-wc-for-view
-  "Takes view-map and action (HoneySQL hash-map for insert/update/delete),
-   extracts where clause from action, and formats it with the proper
-   alias (or no alias) so that it will work when applied to the view SQL."
-  [view-map action]
-  (if (:where action)
-    (let [preds (create-view-delta-where-clauses view-map action)]
-      (if (> (count preds) 1)
-        (into [:or] preds)
-        (first preds)))))
-
-;; DELTA CALCULATIONS
-
-(defn- calculate-delete-deltas
-  [db view-map]
-  (->> (:delete-deltas-map view-map)
-       hsql/format
-       (j/query db)
-       (assoc view-map :delete-deltas)))
-
-(defn compute-delete-deltas-for-insert
-  "Computes and returns a sequence of delete deltas for a single view and insert."
-  [schema db view-map table record]
-  (if (vh/outer-join-table? (:view view-map) table)
-    (let [delta-q (vh/outer-join-delta-query schema (:view view-map) table record)]
-      (j/query db (hsql/format delta-q)))
-    []))
-
-(defn primary-key-predicate
-  "Return a predicate for a where clause that constrains to the primary key of
-  the record."
-  [schema table record]
-  (let [pkey (get-primary-key schema table)]
-    [:= pkey (pkey record)]))
-
-(defn compute-insert-deltas-for-insert
-  [schema db view-map table record]
-  (let [pkey-pred        (primary-key-predicate schema table record)
-        action           (hsql/build :insert-into table :values [record] :where pkey-pred)
-        insert-delta-wc  (format-action-wc-for-view view-map action)
-        view             (:view view-map)
-        insert-delta-map (update-in view [:where] #(:where (vh/merge-where-clauses insert-delta-wc %)))]
-    (j/query db (hsql/format insert-delta-map))))
-
-(defn compute-insert-delete-deltas-for-views
-  [schema db views table record]
-  (doall (map #(compute-delete-deltas-for-insert schema db % table record) views)))
-
-(defn compute-insert-insert-deltas-for-views
-  [schema db views table record]
-  (doall (map #(compute-insert-deltas-for-insert schema db % table record) views)))
-
-(defn compute-deltas-for-insert
-  "This takes a *single* insert and a view, applies the insert and computes
-  the view deltas."
-  [schema db views table record]
-  (let [deletes (compute-insert-delete-deltas-for-views schema db views table record)
-        record* (first (j/insert! db table record))
-        inserts (compute-insert-insert-deltas-for-views schema db views table record*)]
-    {:views-with-deltas (doall (map #(assoc %1 :delete-deltas %2 :insert-deltas %3) views deletes inserts))
-     :result record*}))
-
-(defn- insert-and-append-deltas!
-  "Handles insert and calculation of insert (after insert) delta."
-  [schema db views action table pkey]
-  (let [table (:insert-into action)]
-    (reduce
-     #(-> %1
-          (update-in [:views-with-deltas] into (:views-with-deltas %2))
-          (update-in [:result-set] conj (:result %2)))
-     {:views-with-deltas [] :result-set []}
-     (map #(compute-deltas-for-insert schema db views table %) (:values action)))))
-
-(defn- calculate-insert-deltas
-  "This is for insert deltas for non-insert updates.
-
-  Takes the HoneySQL map (at key :view) from the view-map and appends
-  the appropriately-table-namespaced where clause which limits the
-  view query to the previously inserted or updated records."
-  [db action pkey-wc view-map]
-  (let [action           (assoc action :where pkey-wc)
-        insert-delta-wc  (format-action-wc-for-view view-map action)
-        view             (:view view-map)
-        insert-delta-map (update-in view [:where] #(:where (vh/merge-where-clauses insert-delta-wc %)))
-        deltas           (j/query db (hsql/format insert-delta-map))]
-    (if (seq deltas)
-      (update-in view-map [:insert-deltas] #(apply conj % deltas))
-      view-map)))
-
-(defn- get-action-row-key
-  "Helper to query the action's table for primary key and pull it out."
-  [db pkey table action]
-  (->> (:where action)
-       (hsql/build :select pkey :from table :where)
-       hsql/format
-       (j/query db)
-       first pkey))
-
-(defn- update-and-append-deltas!
-  "Handles update and calculation of delete (before update) and insert (after update) deltas."
-  [db views action table pkey]
-  (let [views-pre (doall (map #(calculate-delete-deltas db %) views))
-        pkey-val  (get-action-row-key db pkey table action)
-        update    (j/execute! db (hsql/format action))]
-    {:views-with-deltas (doall (map #(calculate-insert-deltas db action [:= pkey pkey-val] %) views-pre))
-     :result-set        update}))
-
-(defn- delete-and-append-deltas!
-  "Handles deletion and calculation of delete (before update) delta."
-  [db views action table pkey]
-  (let [views-pre (doall (map #(calculate-delete-deltas db %) views))]
-    {:views-with-deltas views-pre
-     :result-set        (j/execute! db (hsql/format action))}))
-
-(defn perform-action-and-return-deltas
-  "Identifies which action--insert, update or delete--we are performing and dispatches appropriately.
-  Returns view-map with appropriate deltas appended."
-  [schema db views action table pkey]
-  (cond
-   (:insert-into action) (insert-and-append-deltas! schema db views action table pkey)
-   (:update action)      (update-and-append-deltas! db views action table pkey)
-   (:delete-from action) (delete-and-append-deltas! db views action table pkey)
-   :else (throw (Exception. "Received malformed action: " action))))
-
-(defn generate-view-delta-map
-  "Adds a HoneySQL hash-map for the delta-calculation specific to the view + action.
-   Takes a view-map and the action HoneySQL hash-map, and appends the action's
-   where clause to the view's where clause, and adds in new field :insert-deltas-map."
-  [view-map action]
-  (let [action-wc (format-action-wc-for-view view-map action)
-        view      (:view view-map)]
-    (->> (update-in view [:where] #(:where (vh/merge-where-clauses action-wc %)))
-         (assoc view-map :delete-deltas-map))))
-
-(defn update-deltas-with-refresh-set
-  [refresh-set]
-  (fn [view-deltas]
-    (if (coll? view-deltas)
-      (map #(assoc % :refresh-set refresh-set) view-deltas)
-      [{:refresh-set refresh-set}])))
-
-(defn calculate-refresh-sets
-  "For refresh-only views, calculates the refresh-set and adds it to the view's delta update collection."
-  [deltas db templates refresh-only-views]
-  (reduce
-   (fn [d {:keys [view-sig view] :as rov}]
-     (let [refresh-set (get (vdbl/initial-view db view-sig templates view) view-sig)]
-       (update-in d [view-sig] (update-deltas-with-refresh-set refresh-set))))
-   deltas
-   refresh-only-views))
-
-(defn format-deltas
-  "Removes extraneous data from view delta response collections.
-  TODO: Is there only one delta pack per view-sig here?"
-  [views-with-deltas]
-  (reduce #(update-in %1 [(:view-sig %2)] (fnil conj []) (select-keys %2 [:delete-deltas :insert-deltas :refresh-set]))
-          {} views-with-deltas))
-
-(defn do-view-transaction
-  "Takes the following arguments:
-   schema    - from edl.core/defschema
-   db        - clojure.java.jdbc database connection
-   all-views - the current set of views (view-maps--see view-map fn docstring for
-                  description) in memory for the database
-   action    - the HoneySQL pre-SQL hash-map with parameters already interpolated.
-   templates - the mapping of view names (keywords) to SQL templates
-                  (a.k.a. HoneySQL hash-map producing functions)
-
-   The function will then perform the following sequence of actions, all run
-   within a transaction (with isolation serializable):
-
-   1) Create pre-check SQL for each view in the list.
-   2) Run the pre-check SQL (or fail out based on some simple heuristics) to
-      identify if we want to send delta messages to the view's subscribers
-      (Note: this happens after the database action for *inserts only*).
-   3) Run the database action (insert/action/delete).
-   4) Calculate deltas based on the method described in section 5.4, \"Rule Generation\"
-      of the paper \"Deriving Production Rules for Incremental Rule Maintenance\"
-      by Stefano Ceri and Jennifer Widom (http://ilpubs.stanford.edu:8090/8/1/1991-4.pdf)
-
-   The function returns a hash-map with :result-set and :new-deltas collection values.
-   :new-deltas contains :insert-deltas, :delete-deltas, and :refresh-set values, as well
-   as the original :view-sig the deltas apply to."
-  [schema db all-views action templates]
-  (j/with-db-transaction [t db :isolation :serializable]
-    (let [filtered-views     (filterv #(vc/have-overlapping-tables? action (:view %)) all-views)
-          {full-refresh-views true normal-views nil} (group-by :refresh-only? filtered-views)
-          need-deltas        (map #(generate-view-delta-map % action) normal-views)
-          table              (-> action vh/extract-tables ffirst)
-          pkey               (get-primary-key schema table)
-          {:keys [views-with-deltas result-set]} (perform-action-and-return-deltas schema t need-deltas action table pkey)
-          deltas             (calculate-refresh-sets (format-deltas views-with-deltas) t templates full-refresh-views)]
-      {:new-deltas deltas :result-set result-set})))

src/views/db/honeysql.clj

@@ -1,148 +0,0 @@
-(ns views.db.honeysql
-  (:require
-    [honeysql.core :as hsql]
-    [honeysql.helpers :as hh]
-    [clojure.string :refer [split]]))
-
-(def table-clauses
-  [:from :insert-into :update :delete-from :join :left-join :right-join])
-
-(def pred-ops
-  #{:= :< :> :<> :>= :<= :in :between :match :ltree-match :and :or :not=})
-
-(defn process-complex-clause
-  [tables clause]
-  (reduce
-   #(if (coll? %2)
-      (if (some pred-ops [(first %2)])
-        %1
-        (conj %1 %2))
-      (conj %1 [%2]))
-   tables
-   clause))
-
-(defn extract-tables*
-  [tables clause]
-  (if clause
-    (if (coll? clause)
-      (process-complex-clause tables clause)
-      (conj tables [clause]))
-    tables))
-
-(defn with-op
-  "Takes a collection of things and returns either an nary op of them, or
-  the item in the collection if there is only one."
-  [op coll]
-  (if (> (count coll) 1) (into [op] coll) (first coll)))
-
-(defn extract-tables
-  "Extracts a set of table vector from a HoneySQL spec hash-map.
-   Each vector either contains a single table keyword, or the
-   table keyword and an alias keyword."
-  ([hh-spec] (extract-tables hh-spec table-clauses))
-  ([hh-spec clauses] (reduce #(extract-tables* %1 (%2 hh-spec)) #{} clauses)))
-
-(defn find-table-aliases
-  "Returns the table alias for the supplied table."
-  [action-table tables]
-  (filter #(= (first action-table) (first %)) tables))
-
-(defn outer-join-table?
-  "Return true if table is used in an outer join in the honeysql expression."
-  [hh-spec table]
-  (let [tables (map first (extract-tables hh-spec [:left-join :right-join]))]
-    (boolean (some #(= table %) tables))))
-
-(defn prefix-column
-  "Prefixes a column with an alias."
-  [column alias]
-  (keyword (str (name alias) \. (name column))))
-
-(defn create-null-constraints
-  "Create 'is null' constraints for all the columns of a table."
-  [schema table table-alias]
-  (let [columns (map #(prefix-column % table-alias) (keys (:columns (get schema (name table)))))]
-    (into [:and] (for [c columns] [:= c nil]))))
-
-(defn table-alias
-  "Returns the name of a table or its alias. E.g. for [:table :t] returns :t."
-  [table]
-  (if (keyword? table) table (last table)))
-
-(defn table-name
-  "Returns the name of a table . E.g. for [:table :t] returns :table."
-  [table]
-  (if (keyword? table) table (first table)))
-
-(defn table-column
-  "Assumes that table columns are keywords of the form :table.column. Returns
-  the column as a keyword or nil if the supplied keyword doesn't match the pattern."
-  [table item]
-  (let [s (name item), t (str (name table) \.)]
-    (if (.startsWith s t) (keyword (subs s (count t))))))
-
-(defn modified-outer-join-predicate
-  "Returns an outer join predicate with the join tables columns subistituted
-   with values from a record."
-  [table predicate record]
-  (if-let [column (and (keyword? predicate) (table-column table predicate))]
-    (or (get record column)
-        (throw (Exception. (str "No value for column " column " in " record))))
-    (if (vector? predicate)
-      (apply vector (map #(modified-outer-join-predicate table % record) predicate))
-      predicate)))
-
-(defn find-outer-joins
-  "Find and return all the outer joins on a given table."
-  [hh-spec table]
-  (->> (concat (:left-join hh-spec) (:right-join hh-spec))
-    (partition 2 2)
-    (filter #(= table (table-name (first %))))))
-
-(defn- create-outer-join-predicates
-  "Create outer join predicate from a record and joins."
-  [schema table record joins]
-  (->> joins
-    (map (fn [[table-spec join-pred]]
-           [:and
-            (modified-outer-join-predicate (table-alias table-spec) join-pred record)
-            (create-null-constraints schema table (table-alias table-spec))]))
-    (with-op :or)))
-
-(defn outer-join-delta-query
-  "Create an outer join delta query given a honeysql template and record"
-  [schema hh-spec table record]
-  (let [join-tables (find-outer-joins hh-spec table)
-        join-pred   (create-outer-join-predicates schema table record join-tables)]
-    (assert (not (nil? join-pred)))
-    (update-in hh-spec [:where] #(vector :and % join-pred))))
-
-(defn merge-where-clauses
-  "Takes two where clauses from two different HoneySQL maps and joins then with an and.
-   If one is nil, returns only the non-nil where clause."
-  [wc1 wc2]
-  (if (and wc1 wc2)
-    (hh/where wc1 wc2)
-    (hh/where (or wc1 wc2))))
-
-(defn replace-table
-  "Replace all instances of table name t1 pred with t2."
-  [pred t1 t2]
-  (if-let [column (and (keyword? pred) (table-column t1 pred))]
-    (keyword (str (name t2) \. (name column)))
-    (if (coll? pred)
-      (map #(replace-table % t1 t2) pred)
-      pred)))
-
-(defn unprefixed-column?
-  [c]
-  (and (keyword? c) (not (pred-ops c)) (neg? (.indexOf (name c) (int \.)))))
-
-(defn prefix-columns
-  "Prefix all unprefixed columns with table."
-  [pred table]
-  (if (unprefixed-column? pred)
-    (keyword (str (name table) \. (name pred)))
-    (if (coll? pred)
-      (map #(prefix-columns % table) pred)
-      pred)))

src/views/db/load.clj

@@ -1,28 +0,0 @@
-(ns views.db.load
-  (:require
-   [clojure.tools.logging :refer [debug info warn error]]
-   [clojure.java.jdbc :as j]
-   [honeysql.core :as hsql]))
-
-(defn view-query
-  "Takes db and query-fn (compiled HoneySQL hash-map)
-   and runs the query, returning results."
-  [db query-map]
-  (j/query db (hsql/format query-map)))
-
-(defn post-process-result-set
-  [view-sig templates result-set]
-  (if-let [post-fn (get-in templates [(first view-sig) :post-fn])]
-    (mapv post-fn result-set)
-    result-set))
-
-(defn initial-view
-  "Takes a db spec, the new views sigs (new-views) we want to produce result-sets for,
-   the template config map, and the view-map itself.
-   and returns a result-set for the new-views with post-fn functions applied to the data."
-  [db new-view templates view-map]
-  (->> view-map
-       (view-query db)
-       (into [])
-       (post-process-result-set new-view templates)
-       (hash-map new-view)))

src/views/db/util.clj

@@ -1,41 +0,0 @@
-(ns views.db.util
-  (:import
-    [java.sql SQLException])
-  (:require
-    [clojure.tools.logging :refer [debug]]))
-
-;; Need to catch this and retry:
-;; java.sql.SQLException: ERROR: could not serialize access due to concurrent update
-;;
-(defn get-nested-exceptions*
-  [exceptions e]
-  (if-let [next-e (.getNextException e)]
-    (recur (conj exceptions next-e) next-e)
-    exceptions))
-
-(defn get-nested-exceptions
-  "Return the current exception and all nested exceptions as a vector."
-  [e]
-  (get-nested-exceptions* [e] e))
-
-;; TODO: update to avoid stack overflow.
-(defn retry-on-transaction-failure
-  "Retry a function whenever we receive a transaction failure."
-  [transaction-fn]
-  (try
-    (transaction-fn)
-    (catch SQLException e
-      ;; http://www.postgresql.org/docs/9.2/static/errcodes-appendix.html
-      (debug "Caught exception with error code: " (.getSQLState e))
-      (debug "Exception message: " (.getMessage e))
-
-      ;; (debug "stack trace message: " (.printStackTrace e))
-      (if (some #(= (.getSQLState %) "40001") (get-nested-exceptions e))
-        (retry-on-transaction-failure transaction-fn) ;; try it again
-        (throw e))))) ;; otherwise rethrow
-
-(defmacro with-retry
-  "Retry a transaction forever."
-  [ & body]
-  `(let [tfn# (fn [] ~@body)]
-    (retry-on-transaction-failure tfn#)))

src/views/filters.clj

@@ -1,36 +0,0 @@
-(ns views.filters
-  (:require
-   [clojure.tools.logging :refer [debug info warn error]]))
-
-(defn view-filter
-  "Takes a subscription request msg, a collection of view-sigs and
-   the config templates hash-map for an app. Checks if there is
-   a global filter-fn in the hash-map metadata and checks against
-   that if it exists, as well as against any existing filter
-   functions for individual template config entries. Template
-   config hash-map entries can specify a filter-fn using the key
-   :filter-fn, and the global filter-fn is the same, only on
-   the config meta-data (i.e. (with-meta templates {:filter-fn ...}))
-
-   By default throws an exception if no filters are present.
-   By passing in {:unsafe true} in opts, this can be overridden."
-  [msg view-sigs templates & opts]
-  (let [global-filter-fn (:filter-fn (meta templates))]
-    (filterv
-     #(let [filter-fn (:filter-fn (get templates (first %)))]
-        (cond
-         (and filter-fn global-filter-fn)
-         (and (global-filter-fn msg %) (filter-fn msg %))
-
-         filter-fn
-         (filter-fn msg %)
-
-         global-filter-fn
-         (global-filter-fn msg %)
-
-         :else
-         (if (-> opts first :unsafe?)
-           (do (warn "YOU ARE RUNNING IN UNSAFE MODE, AND NO FILTERS ARE PRESENT FOR VIEW-SIG: " %)
-               true)
-           (throw (Exception. (str "No filter set for view " %))))))
-     view-sigs)))

src/views/persistence/core.clj

@@ -1,25 +0,0 @@
-(ns views.persistence.core)
-
-(defprotocol IPersistence
-  (subscribe! [this templates namespace view-sig subscriber-key]
-    "Subscribes a subscriber with subscriber-key to a view with signature
-     view-sig. Templates is a map of all defined view templates and db
-     is a jdbc transcation handle for the database from which initial
-     view data will be retrieved.
-
-     This function must return the view-data for the subscribed view.")
-
-  (unsubscribe! [this namespace view-sig subscriber-key]
-    "Unsubscribes a subscriber with key 'subscriber-key' from the view
-    with signature 'view-sig' in namespace 'namespace'.")
-
-  (unsubscribe-all! [this namespace subscriber-key]
-    "Unsubscribes the subscriber with key 'subscriber-key' from ALL views
-    in namespace 'namespace'.")
-
-  (view-data [this namespace table-name]
-    "Return all the view data that references a table name in a namespace.")
-
-  (subscriptions [this namespace signatures]
-   "Return all subscribers for all signatures in the list 'signatures' in
-   a namespace."))

src/views/persistence/memory.clj

@@ -1,62 +0,0 @@
-(ns views.persistence.memory
-  (:require
-    [views.persistence.core :refer :all]
-    [views.db.deltas :as vd]))
-
-(defn ns-subscribe!
-  "Subscribe to a view inside a namespace."
-  [namespace-views view-sig templates subscriber-key]
-  (-> namespace-views
-      (update-in [view-sig :subscriptions] (fnil conj #{}) subscriber-key)
-      (assoc-in  [view-sig :view-data]     (vd/view-map (get-in templates [(first view-sig) :fn]) view-sig))))
-
-(defn ns-unsubscribe!
-  "Unsubscribe from a view inside a namespace. If there are no more subscribers,
-  we remove the view itself as well."
-  [namespace-views view-sig subscriber-key]
-  (let [path    [view-sig :subscriptions]
-        updated (update-in namespace-views path disj subscriber-key)]
-    (if (seq (get-in updated path))
-      updated
-      (dissoc updated view-sig))))
-
-(defn ns-unsubscribe-all!
-  "Unsubscribe a subscriber from all views in a namespace."
-  [namespace-views subscriber-key]
-  (reduce #(ns-unsubscribe! %1 %2 subscriber-key) namespace-views (keys namespace-views)))
-
-(defn ns-subscriptions
-  "Find subscribers for a signature and add to a map."
-  [namespace-views result-map sig]
-  (if-let [subscribers (get-in namespace-views [sig :subscriptions])]
-    (assoc result-map sig subscribers)
-    result-map))
-
-(deftype ViewsMemoryPersistence [subbed-views]
-  IPersistence
-  (subscribe!
-    [this templates namespace view-sig subscriber-key]
-    (let [sv (swap! subbed-views (fn [sv] (update-in sv [namespace] ns-subscribe! view-sig templates subscriber-key)))]
-      (get-in sv [namespace view-sig :view-data])))
-
-  (unsubscribe!
-    [this namespace view-sig subscriber-key]
-    (swap! subbed-views
-           (fn [sv] (update-in sv [namespace] ns-unsubscribe! view-sig subscriber-key))))
-
-  (unsubscribe-all!
-    [this namespace subscriber-key ]
-    (swap! subbed-views
-           (fn [sv] (update-in sv [namespace] ns-unsubscribe-all! subscriber-key))))
-
-  (view-data [this namespace table]
-    ;; We don't yet use table name as an optimization here.
-    (map :view-data (vals (get @subbed-views namespace))))
-
-  (subscriptions [this namespace signatures]
-    (let [namespace-views (get @subbed-views namespace)]
-      (reduce #(ns-subscriptions namespace-views %1 %2) {} signatures))))
-
-(defn new-memory-persistence
-  []
-  (->ViewsMemoryPersistence (atom {})))

src/views/protocols.clj

@@ -0,0 +1,14 @@
+(ns views.protocols)
+
+(defprotocol IView
+  (data [this namespace parameters]
+    "Returns view data.")
+  (relevant? [this namespace parameters hints]
+    "Given hints of the form {:namespace x :hint y :type z}, the view must
+     return true if the hint indicates that an instance of this view
+     with supplied namespace and parameters might require updating.
+     It is always safe to return true, but false should be returned only
+     if you are sure this view does not need updating.")
+  (id [this]
+    "A unique identifer for a view."))
+

src/views/riemann.clj

@@ -1,5 +0,0 @@
-(ns views.riemann
-  (:require
-   [riemann.client :refer [tcp-client]]))
-
-(defonce rclient (tcp-client {:host "127.0.0.1" :port 5555}))

src/views/router.clj

@@ -1,39 +0,0 @@
-(ns views.router
-  (:require
-   [views.subscribed-views :refer [subscribe-views unsubscribe-views disconnect]]
-   [clojure.core.async :refer [go go-loop chan pub sub unsub close! >! >!! <! <!! filter<]]
-   [clojure.tools.logging :refer [debug]]))
-
-(defn handle-subscriptions!
-  [subscribed-views subscriptions]
-  (go (while true
-        (let [sub (<! subscriptions)]
-          (debug "Subscribing (in router): " sub)
-          (subscribe-views subscribed-views sub)))))
-
-(defn handle-unsubscriptions!
-  [subscribed-views unsubscriptions]
-  (go (while true
-        (let [unsub (<! unsubscriptions)]
-          (debug "Unsubscribing (in router): " unsub)
-          (unsubscribe-views subscribed-views unsub)))))
-
-(defn handle-disconnects!
-  [subscribed-views disconnects]
-  (go (while true
-        (let [disc (<! disconnects)]
-          (debug "Disconnecting (in router): " disc)
-          (disconnect subscribed-views disc)))))
-
-(defn init!
-  [{:keys [base-subscribed-views] :as conf} client-chan]
-  (let [subs        (chan 200)
-        unsubs      (chan 200)
-        control     (chan 200)
-        disconnects (filter< #(= :disconnect (:body %)) control)]
-    (sub client-chan :views.subscribe subs)
-    (sub client-chan :views.unsubscribe unsubs)
-    (sub client-chan :client-channel disconnects)
-    (handle-subscriptions! base-subscribed-views subs)
-    (handle-unsubscriptions! base-subscribed-views unsubs)
-    (handle-disconnects! base-subscribed-views disconnects)))

src/views/subscribed_views.clj

@@ -1,11 +0,0 @@
-(ns views.subscribed-views)
-
-(defprotocol ISubscribedViews
-  ;; Subscription and Delta routing
-  (subscribe-views [this sub-request])
-  (unsubscribe-views [this unsub-request])
-  (disconnect [this disconnect-request])
-
-  ;; DB interaction
-  (subscribed-views [this namespace])
-  (broadcast-deltas [this deltas namespace]))

test/views/base_subscribed_views_test.clj

@@ -1,143 +0,0 @@
-(ns views.base-subscribed-views-test
-  (:require
-   [views.base-subscribed-views :as bsv]
-   [views.persistence.core :refer :all]
-   [views.persistence.memory :refer [new-memory-persistence]]
-   [views.subscribed-views :refer [subscribe-views unsubscribe-views disconnect broadcast-deltas]]
-   [views.fixtures :as vf]
-   [clojure.test :refer [use-fixtures deftest is]]
-   [clojure.java.jdbc :as j]
-   [clj-logging-config.log4j :refer [set-logger! set-loggers!]])
-  (:import
-   [views.base_subscribed_views BaseSubscribedViews]))
-
-(set-loggers!
- "views.base-subscribed-views" {:level :error}
- "views.filters"               {:level :error})
-
-(defn view-config []
-  {:persistence (new-memory-persistence)
-   :db vf/db
-   :templates vf/templates
-   :view-sig-fn :views
-   :unsafe? true})
-
-(deftest subscribes-and-dispatches-initial-view-result-set
-  (let [config (view-config)
-        sent   (atom #{})
-        send-fn #(do (is (and (= %1 1) (= %2 :views.init) (= %3 {[:users] []})))
-                     (swap! sent conj [%1 %2 %3]))
-        base-subbed-views (BaseSubscribedViews. (assoc config :send-fn send-fn))]
-    (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]})
-    (Thread/sleep 10)
-    (is (= (subscriptions (:persistence config) bsv/default-ns [[:users]])
-           {[:users] #{1}}))
-    ;; Verify sends occured.
-    (is (= @sent #{[1 :views.init {[:users] []}]}))))
-
-;; This test illustrates a slight timing issue. Because view subscriptions
-;; use threads, an unsubscription that follows a subscription too closely
-;; can fail.
-(deftest unsubscribes-view
-  (let [config            (view-config)
-        base-subbed-views (BaseSubscribedViews. config)]
-    (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]})
-    (Thread/sleep 10)
-    (unsubscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]})
-    (is (= (subscriptions (:persistence config) bsv/default-ns [[:users]])
-           {}))))
-
-(deftest filters-subscription-requests
-  (let [config            (view-config)
-        templates         (assoc-in vf/templates [:users :filter-fn]
-                                    (fn [msg _] (:authorized? msg)))
-        view-config (-> config (assoc :templates templates) (dissoc :unsafe?))
-        base-subbed-views (BaseSubscribedViews. view-config)]
-    (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users]]})
-    (Thread/sleep 10)
-    (is (= (subscriptions (:persistence config) bsv/default-ns [[:users]])
-           {}))))
-
-(deftest removes-all-subscriptions-on-disconnect
-  (let [config            (view-config)
-        base-subbed-views (BaseSubscribedViews. config)]
-    (subscribe-views base-subbed-views {:subscriber-key 1 :views [[:users] [:user-posts 1]]})
-    (Thread/sleep 10)
-    (is (= (subscriptions (:persistence config) bsv/default-ns [[:users] [:user-posts 1]])
-           {[:users] #{1}, [:user-posts 1] #{1}}))
-    (disconnect base-subbed-views {:subscriber-key 1})
-    (is (= (subscriptions (:persistence config) bsv/default-ns [[:users] [:user-posts 1]])
-           {}))))
-
-;; (deftest sends-deltas
-;;   (let [deltas {[:users] [{:view-sig [:users] :insert-deltas [{:foo "bar"}]}]}
-;;         sent-delta {[:users] {:insert-deltas [{:foo "bar"}]}}
-;;         send-fn #(do (is (#{1 2} %1))
-;;                      (is (= %2 :views.deltas))
-;;                      (is (= %3 sent-delta)))
-;;         base-subbed-views (BaseSubscribedViews. (assoc view-config-fixture :send-fn send-fn))]
-;;     (add-subscription! [:users] vf/templates 1 default-ns)
-;;     (add-subscription! [:users] vf/templates 2 default-ns)
-;;     (broadcast-deltas base-subbed-views deltas nil)))
-
-(deftest sends-deltas-in-batch
-  (let [config (view-config)
-        deltas [{[:users] [{:insert-deltas [{:id 1 :name "Bob"} {:id 2 :name "Alice"}]}]}
-                {[:users] [{:insert-deltas [{:id 3 :name "Jack"} {:id 4 :name "Jill"}]}]}]
-        ;; This is just more obvious than writing some convulated fn to dig out the view-sigs.
-        sent-deltas [{[:users] [{:insert-deltas [{:id 1 :name "Bob"} {:id 2 :name "Alice"}]}]}
-                     {[:users] [{:insert-deltas [{:id 3 :name "Jack"} {:id 4 :name "Jill"}]}]}]
-        sent    (atom #{})
-        send-fn #(do (is (#{1 2} %1))
-                     (is (= :views.deltas %2))
-                     (is (= sent-deltas %3))
-                     (swap! sent conj [%1 %2 %3]))
-        base-subbed-views (BaseSubscribedViews. (assoc config :send-fn send-fn))]
-    (subscribe! (:persistence config) vf/templates bsv/default-ns [:users] 1)
-    (broadcast-deltas base-subbed-views deltas nil)
-    (is (= 1 (count @sent)))
-    (is (= 1 (ffirst @sent)))
-    (is (= :views.deltas (second (first @sent))))
-    (is (= sent-deltas (nth (first @sent) 2)))))
-
-(deftest deltas-are-post-processed
-  (let [config      (view-config)
-        templates   (assoc-in vf/templates [:users :post-fn] (fn [d] (update-in d [:id] #(Integer. %))))
-        deltas      [{[:users] [{:insert-deltas [{:id "1" :name "Bob"}]}]}]
-        sent-deltas [{[:users] [{:insert-deltas [{:id "1" :name "Bob"}]}]}]
-        sent        (atom #{})
-        send-fn     (fn [a b deltas-out]
-                      (is (= (:id (first (:insert-deltas (first (get (first deltas-out) [:users])))))
-                             1))
-                      (swap! sent conj [a b deltas-out]))
-        base-subbed-views (BaseSubscribedViews. (assoc config :send-fn send-fn :templates templates))]
-    (subscribe! (:persistence config) vf/templates bsv/default-ns [:users] 1)
-    (Thread/sleep 10)
-    (broadcast-deltas base-subbed-views deltas nil)
-    (is (= 1 (count @sent)))
-    (is (= 1 (ffirst @sent)))
-    (is (= :views.deltas (second (first @sent))))
-    (is (not= sent-deltas (nth (first @sent) 2)))
-    (is (= [{[:users] [{:insert-deltas [{:name "Bob", :id 1}]}]}] (nth (first @sent) 2)))))
-
-(deftest full-refresh-deltas-are-post-processed
-  (let [config      (view-config)
-        templates   (assoc-in vf/templates [:users :post-fn] (fn [d] (update-in d [:id] #(Integer. %))))
-        deltas      [{[:users] [{:refresh-set [{:id "1" :name "Bob"}]}]}]
-        sent-deltas [{[:users] [{:refresh-set [{:id "1" :name "Bob"}]}]}]
-        sent        (atom #{})
-        send-fn     (fn [a b deltas-out]
-                      (is (= (:id (first (:refresh-set (first (get (first deltas-out) [:users])))))
-                             1))
-                      (swap! sent conj [a b deltas-out]))
-        base-subbed-views (BaseSubscribedViews. (assoc config :send-fn send-fn :templates templates))]
-    (subscribe! (:persistence config) vf/templates bsv/default-ns [:users] 1)
-    (Thread/sleep 10)
-    (broadcast-deltas base-subbed-views deltas nil)
-    (is (= 1 (count @sent)))
-    (is (= 1 (ffirst @sent)))
-    (is (= :views.deltas (second (first @sent))))
-    (is (not= sent-deltas (nth (first @sent) 2)))
-    (is (= [{[:users] [{:refresh-set [{:name "Bob", :id 1}]}]}] (nth (first @sent) 2)))))
-
-

test/views/basic_system_init_tests.clj

@@ -0,0 +1,113 @@
+(ns views.basic-system-init-tests
+  (:use
+    clojure.test
+    views.test-helpers
+    views.protocols
+    views.core
+    views.test-view-system)
+  (:import
+    (views.test_view_system MemoryView)
+    (clojure.lang Atom)))
+
+(use-fixtures :each reset-test-views-system)
+
+
+(defn dummy-send-fn [subscriber-key [view-sig view-data]])
+
+(def test-options (merge default-options
+                         {:views views
+                          :send-fn dummy-send-fn}))
+
+
+;; tests
+
+(deftest inits-with-correct-config-and-shutsdown-correctly
+  (let [options test-options
+        ; 1. init views
+        init-returned-atom (init! test-views-system test-options)]
+    (is (instance? Atom init-returned-atom))
+    (is (= init-returned-atom test-views-system))
+    (is (seq @test-views-system))
+    (is (= dummy-send-fn (:send-fn @test-views-system)))
+    (is (and (contains-view? test-views-system :foo)
+             (contains-view? test-views-system :bar)
+             (contains-view? test-views-system :baz)))
+    (is (not (:logging? @test-views-system)))
+    (is (not (collecting-stats? test-views-system)))
+    (is (empty? (subscribed-views test-views-system)))
+    (let [refresh-watcher (:refresh-watcher @test-views-system)
+          workers         (:workers @test-views-system)]
+      (is (.isAlive ^Thread refresh-watcher))
+      (is (= (:worker-threads options) (count workers)))
+      (doseq [^Thread t workers]
+        (is (.isAlive t)))
+      ; 2. shutdown views (and wait for all threads to also finish)
+      (shutdown! test-views-system)
+      (is (empty? @test-views-system))
+      (is (not (.isAlive ^Thread refresh-watcher)))
+      (doseq [^Thread t workers]
+        (is (not (.isAlive t)))))))
+
+(deftest init-without-existing-view-system-atom
+  (let [options test-options]
+    (let [init-created-atom (init! options)]
+      (is (instance? Atom init-created-atom))
+      (is (seq @init-created-atom))
+      (is (= dummy-send-fn (:send-fn @init-created-atom)))
+      (is (and (contains-view? init-created-atom :foo)
+               (contains-view? init-created-atom :bar)
+               (contains-view? init-created-atom :baz)))
+      (shutdown! init-created-atom)
+      (is (empty? @init-created-atom)))))
+
+(deftest init-can-also-start-logger
+  (let [options (-> test-options
+                    (assoc :stats-log-interval 10000))]
+    ; 1. init views
+    (init! test-views-system options)
+    (is (seq (:statistics @test-views-system)))
+    (is (:logging? @test-views-system))
+    (is (collecting-stats? test-views-system))
+    (let [logger-thread (get-in @test-views-system [:statistics :logger])]
+      (is (.isAlive ^Thread logger-thread))
+      ; 2. shutdown views
+      (shutdown! test-views-system)
+      (is (nil? (get-in @test-views-system [:statistics :logger])))
+      (is (not (.isAlive ^Thread logger-thread))))))
+
+(deftest can-add-new-views-after-init
+  (let [options test-options]
+    ; 1. init views
+    (init! test-views-system options)
+    (is (and (contains-view? test-views-system :foo)
+             (contains-view? test-views-system :bar)
+             (contains-view? test-views-system :baz)))
+    ; 2. add new views
+    (add-views! test-views-system
+                [(MemoryView. :one [:one])
+                 (MemoryView. :two [:two])])
+    (is (and (contains-view? test-views-system :foo)
+             (contains-view? test-views-system :bar)
+             (contains-view? test-views-system :baz)
+             (contains-view? test-views-system :one)
+             (contains-view? test-views-system :two)))
+    ; 3. shutdown views
+    (shutdown! test-views-system)))
+
+(deftest can-replace-views-after-init
+  (let [options          test-options
+        replacement-view (MemoryView. :foo [:new-foo])]
+    ; 1. init views
+    (init! test-views-system options)
+    (is (and (contains-view? test-views-system :foo)
+             (contains-view? test-views-system :bar)
+             (contains-view? test-views-system :baz)))
+    (is (not= replacement-view (get-in @test-views-system [:views :foo])))
+    ; 2. add view. has same id so should replace existing one
+    (add-views! test-views-system [replacement-view])
+    (is (and (contains-view? test-views-system :foo)
+             (contains-view? test-views-system :bar)
+             (contains-view? test-views-system :baz)))
+    (is (= replacement-view (get-in @test-views-system [:views :foo])))
+    ; 3. shutdown views
+    (shutdown! test-views-system)))

test/views/core_test.clj

@@ -1,15 +0,0 @@
-(ns views.core-test
-  (:require
-   [clojure.test :refer [use-fixtures deftest is]]
-   [edl.core :refer [defschema]]
-   [views.fixtures :as vf]
-   [views.subscribed-views :as vs]
-   [views.core :refer [config]]))
-
-(defschema schema vf/db "public")
-
-#_(deftest configures-views
-  (let [conf (config {:db vf/db :schema schema :templates vf/templates :unsafe? true})]
-    ;; wtf is this false?! AKH: there is some sort of recursive referencing going on
-    ;; in the thing being compared to.
-    (is (satisfies? views.subscribed-views/ISubscribedViews (:subscribed-views conf)))))

test/views/db/checks_test.clj

@@ -1,23 +0,0 @@
-(ns views.db.checks-test
-  (:require
-   [clojure.test :refer [deftest is run-tests]]
-   [honeysql.core :as hsql]
-   [honeysql.helpers :as hh]
-   [views.fixtures :as vf]
-   [views.db.checks :as vc]))
-
-(defn view [a b] (hsql/build :select [:c :d :f] :from {:foo :f} :where [:and [:and [:= :a a] [:= :b b]]]))
-
-(deftest swaps-predicates-and-extracts-clauses
-  (let [{:keys [p q]} (vc/swap-preds (view "?1" "?2"))
-        swapped {:where [:and [:and true true]], :from {:foo :f}, :select [:c :d :f]}]
-    (is (= (set p) #{[:= :a "?1"] [:= :b "?2"]}))
-    (is (= (:where q) (:where swapped)))))
-
-(deftest constructs-view-check
-  (let [dummy-vm (apply view (vc/view-sig->dummy-args [:view 1 2]))
-        update   (hsql/build :update :foo :set {:d "d"} :where [:= :c "c"])
-        check    (hsql/build :select [:a :b] :from :foo :where [:and [:and true true] [:= :c "c"]])
-        calcc    (vc/view-check update dummy-vm)] ;;view )]
-    (is (= (into #{} (:select check)) (into #{} (:select calcc))))
-    (is (= (:where check) (:where calcc)))))

test/views/db/core_test.clj

@@ -1,66 +0,0 @@
-(ns views.db.core-test
-  (:require
-   [clojure.test :refer [use-fixtures deftest is]]
-   [views.persistence.core :as persist]
-   [views.persistence.memory :refer [new-memory-persistence]]
-   [views.base-subscribed-views :refer [default-ns]]
-   [views.subscribed-views :refer [ISubscribedViews]]
-   [views.fixtures :as vf :refer [vschema sql-ts]]
-   [views.db.core :as vdb]))
-
-(def received-deltas (atom nil))
-(def memory (atom (new-memory-persistence)))
-
-;; Very barebones subscribed-views instance which merely satisfies what vexec! needs:
-(deftype TestSubscribedViews []
-  ISubscribedViews
-  (subscribed-views [this namespace]
-    (persist/view-data @memory default-ns nil))
-
-  (broadcast-deltas [this new-deltas namespace]
-    (reset! received-deltas new-deltas)))
-
-(def test-subscribed-views (TestSubscribedViews.))
-(def test-config {:db vf/db :schema vschema :templates vf/templates :base-subscribed-views test-subscribed-views})
-
-(defn reset-fixtures!
-  [f]
-  (reset! memory (new-memory-persistence))
-  (reset! received-deltas {})
-  (f))
-
-(use-fixtures :each vf/with-user-fixture! (vf/database-fixtures! [:posts :comments]) reset-fixtures!)
-(use-fixtures :once (vf/database-fixtures! [:users]))
-
-(deftest vexec-sends-deltas
-  (let [view-sig     [:user-posts (:id @vf/user-fixture)]
-        sub-to-it    (persist/subscribe! @memory vf/templates default-ns view-sig (:id @vf/user-fixture))
-        posted       (first (vdb/vexec! test-config (vf/insert-post-tmpl (:id @vf/user-fixture) "title" "body")))
-        delta-vs     (ffirst (first @received-deltas))
-        insert-delta (-> @received-deltas ffirst second first :insert-deltas first)]
-
-    (is (= (ffirst (first @received-deltas)) view-sig))
-    (is (= (:name insert-delta) (:name @vf/user-fixture)))
-    (is (= (:body insert-delta) (:body posted)))
-    (is (= (:title insert-delta) (:title posted)))))
-
-(deftest with-view-transaction-sends-deltas
-  (let [view-sig     [:user-posts (:id @vf/user-fixture)]
-        sub-to-it    (persist/subscribe! @memory vf/templates default-ns view-sig (:id @vf/user-fixture))
-        posted       (first (vdb/with-view-transaction
-                              [tc test-config]
-                              (vdb/vexec! tc (vf/insert-post-tmpl (:id @vf/user-fixture) "title" "body"))))
-        delta-vs     (ffirst (first @received-deltas))
-        insert-delta (-> @received-deltas ffirst second first :insert-deltas first)]
-
-    (is (= (ffirst (first @received-deltas)) view-sig))
-    (is (= (:name insert-delta) (:name @vf/user-fixture)))
-    (is (= (:body insert-delta) (:body posted)))
-    (is (= (:title insert-delta) (:title posted)))))
-
-;; (deftest removes-nil-deltas
-;;   (let [deltas {[:foo 1] {:insert-deltas '() :delete-deltas []}
-;;                 [:bar 2] {:insert-deltas '() :delete-deltas [] :refresh-set []}
-;;                 [:baz 2] {:insert-deltas '() :delete-deltas [{:baz 1}]}}]
-;;     (is (= #{[:bar 2] [:baz 2]} (into #{} (keys (vdb/remove-nil-deltas deltas)))))
-;; ))

test/views/db/deltas_test.clj

@@ -1,77 +0,0 @@
-(ns views.db.deltas-test
-  (:require
-   [clojure.test :refer [use-fixtures deftest is]]
-   [honeysql.core :as hsql]
-   [honeysql.helpers :as hh]
-   [views.fixtures :as vf :refer [vschema sql-ts]]
-   [views.db.core :as vdb]
-   [views.db.deltas :as vd]))
-
-(defn dvt-helper
-  ([all-views action] (dvt-helper all-views action vf/templates))
-  ([all-views action templates]
-     (vd/do-view-transaction vschema vf/db all-views action templates)))
-
-(use-fixtures :each (vf/database-fixtures!))
-
-(deftest builds-view-map
-  (let [{:keys [view-sig view refresh-only?]} (vd/view-map vf/users-tmpl [:users])]
-    (is (= view-sig [:users]))
-    (is (= view {:from [:users], :select [:id :name :created_on]}))
-    (is (nil? refresh-only?))))
-
-(defn non-nil-values-for-keys?
-  [hm keys]
-  (every? #(% hm) keys))
-
-(deftest calculates-insert-deltas
-  (let [views     [(vd/view-map vf/users-tmpl [:users])]
-        user-args {:name "Test user" :created_on (sql-ts)}
-        insert    (hsql/build :insert-into :users :values [user-args])
-        {:keys [new-deltas result-set]} (dvt-helper views insert)
-        insert-delta (first (:insert-deltas (first (get new-deltas [:users]))))]
-
-    ;; Result set
-    (is (not (nil? (:id (first result-set)))))
-    (is (= user-args (dissoc (first result-set) :id)))
-
-    ;; Deltas
-    (is (= (:name user-args) (:name insert-delta)))
-    (is (= (:created_on user-args) (:created_on insert-delta)))
-    (is (non-nil-values-for-keys? insert-delta (-> views first :view :select)))))
-
-(deftest calculates-delete-deltas
-  (let [views     [(vd/view-map vf/users-tmpl [:users])]
-        user-args {:name "Test user" :created_on (sql-ts)}
-        user      (vf/view-action! (hsql/build :insert-into :users :values [user-args]))
-        delete    (hsql/build :delete-from :users :where [:= :name (:name user-args)])
-        {:keys [new-deltas result-set]} (dvt-helper views delete)
-        delete-delta (first (:delete-deltas (first (get new-deltas [:users]))))]
-
-    ;; Deltas
-    (is (= (:name user-args) (:name delete-delta)))
-    (is (= (:created_on user-args) (:created_on delete-delta)))
-    (is (non-nil-values-for-keys? delete-delta (-> views first :view :select)))))
-
-(deftest calculates-update-deltas
-  (let [views     [(vd/view-map vf/users-tmpl [:users])]
-        user-args {:name "Test user" :created_on (sql-ts)}
-        user      (vf/view-action! (hsql/build :insert-into :users :values [user-args]))
-        new-name  "new name!"
-        update    (hsql/build :update :users :set {:name new-name} :where [:= :name (:name user-args)])
-        {:keys [new-deltas result-set]} (dvt-helper views update)
-        {:keys [insert-deltas delete-deltas]} (first (get new-deltas [:users]))]
-
-    ;; Deltas
-    (is (= (:name user-args) (:name (first delete-deltas))))
-    (is (= new-name (:name (first insert-deltas))))))
-
-(deftest does-not-calculate-deltas-for-unrelated-views
-  (let [views     [(vd/view-map vf/users-tmpl [:users])
-                   (vd/view-map vf/all-comments-tmpl [:all-comments])]
-        user-args {:name "Test user" :created_on (sql-ts)}
-        insert    (hsql/build :insert-into :users :values [user-args])
-        {:keys [new-deltas result-set]} (dvt-helper views insert)]
-
-;;    (is (= (count (insert-deltas new-deltas) 1))
-    (is (nil? (get new-deltas [:all-comments])))))

test/views/db/honeysql_test.clj

@@ -1,71 +0,0 @@
-(ns views.db.honeysql-test
-  (:require
-   [clojure.test :refer [deftest is run-tests]]
-   [views.db.honeysql :as vh]
-   [honeysql.helpers :as hh]))
-
-(def simple-test
-  (-> (hh/select :a)
-      (hh/from :foo)))
-
-(def insert-test
-  (-> (hh/insert-into :foo)
-      (hh/values [{:foo "foo"}])))
-
-(def join-test
-  (-> (hh/select :a)
-      (hh/from :foo)
-      (hh/join :bar [:= :bar.id :foo.bar_id])))
-
-(def join-with-alias-test
-  (-> (hh/select :a)
-      (hh/from :foo)
-      (hh/join [:bar :b] [:= :b.id :foo.bar_id])))
-
-(def join-and-from-with-alias-test
-  (-> (hh/select :a)
-      (hh/from [:foo :f])
-      (hh/join [:bar :b] [:= :b.id :foo.bar_id])))
-
-(deftest extracts-tables-from-specs
-  (is (= (vh/extract-tables simple-test) #{[:foo]}))
-  (is (= (vh/extract-tables insert-test) #{[:foo]}))
-  (is (= (vh/extract-tables join-test) #{[:foo] [:bar]}))
-  (is (= (vh/extract-tables join-with-alias-test) #{[:foo] [:bar :b]}))
-  (is (= (vh/extract-tables join-and-from-with-alias-test) #{[:foo :f] [:bar :b]})))
-
-;; Do we really need to test the new version?
-(deftest merges-where-clauses
-  (is (= (vh/merge-where-clauses [:= :foo 1] [:= :bar 2])
-         {:where [:and [:= :foo 1] [:= :bar 2]]}))
-  #_(is (= (vh/merge-where-clauses [[:= :foo 1]] [:= :bar 2])
-         {:where [:and [:= :foo 1] [:= :bar 2]]}))
-  #_(is (= (vh/merge-where-clauses [[:= :foo 1]] [:and [:= :bar 2] [:not= :baz 3]])
-         {:where [:and [:= :foo 1] [:= :bar 2] [:not= :baz 3]]}))
-  #_(is (= (vh/merge-where-clauses [[:= :foo 1]] [nil])
-         {:where [:= :foo 1]}))
-  #_(is (= (vh/merge-where-clauses [nil] [:= :bar 2])
-         {:where [:= :bar 2]})))
-
-(deftest table-alias-tests
-  (is (= (vh/table-alias [:bar]) :bar))
-  (is (= (vh/table-alias [:bar :a]) :a))
-  (is (= (vh/table-alias :bar) :bar)))
-
-(deftest table-name-tests
-  (is (= (vh/table-name [:bar]) :bar))
-  (is (= (vh/table-name [:bar :a]) :bar))
-  (is (= (vh/table-name :bar) :bar)))
-
-(deftest prefix-columns-tests
-  (is (= (vh/prefix-columns [:= :id 1] :bar) [:= :bar.id 1]))
-  (is (= (vh/prefix-columns [:and [:= :id 1] [:= :val "foo"]] :b)
-         [:and [:= :b.id 1] [:= :b.val "foo"]]))
-  (is (= (vh/prefix-columns [:and [:= :id 1] [:or [:> :x 3] [:= :val "foo"]]] :b)
-         [:and [:= :b.id 1] [:or [:> :b.x 3] [:= :b.val "foo"]]])))
-
-(deftest replace-table-tests
-  (is (= (vh/replace-table [:= :bar.id 1] :bar :b) [:= :b.id 1]))
-  (is (= (vh/replace-table [:= :bar.id 1] :baz :b) [:= :bar.id 1]))
-  (is (= (vh/replace-table [:and [:= :bar.id 1] [:= :bar.val "foo"]] :bar :b)
-         [:and [:= :b.id 1] [:= :b.val "foo"]])))

test/views/db/load_test.clj

@@ -1,25 +0,0 @@
-(ns views.db.load-test
-  (:require
-   [clojure.test :refer [use-fixtures deftest is]]
-   [honeysql.core :as hsql]
-   [views.fixtures :as vf :refer [gen-n-users! database-fixtures! templates]]
-   [views.db.load :as vload]
-   [clojure.string :refer [upper-case]]))
-
-(use-fixtures :each (database-fixtures!))
-
-(defn subscribed-views
-  []
-  {[:users] {:view ((get-in templates [:users :fn]))}})
-
-(deftest initializes-views
-  (let [users (gen-n-users! 2)]
-    (is (= (vload/initial-view vf/db [:users] templates (get-in (subscribed-views) [[:users] :view]))
-           {[:users] users}))))
-
-(deftest post-processes-views
-  (let [users       (gen-n-users! 1)
-        with-postfn (assoc-in templates [:users :post-fn] #(update-in % [:name] upper-case))
-        views-rs    (vload/initial-view vf/db [:users] with-postfn (get-in (subscribed-views) [[:users] :view]))]
-    (is (= (-> (get views-rs [:users]) first :name)
-           (-> users first :name upper-case)))))

test/views/fixtures.clj

@@ -1,102 +0,0 @@
-(ns views.fixtures
-  (:require
-   [environ.core :as e]
-   [clojure.java.jdbc :as j]
-   [honeysql.core :as hsql]
-   [edl.core :refer [defschema]]
-   [clojure.data.generators :as dg]))
-
-(defn sql-ts
-  ([ts] (java.sql.Timestamp. ts))
-  ([] (java.sql.Timestamp. (.getTime (java.util.Date.)))))
-
-(def db {:classname   "org.postgresql.Driver"
-         :subprotocol "postgresql"
-         :subname     (get :views-test-db e/env "//localhost/views_test")
-         :user        (get :views-test-user e/env "views_user")
-         :password    (get :views-test-ppassword e/env "password")})
-
-(defschema vschema db "public")
-
-(defn clean-tables!
-  [tables]
-  (doseq [t (map name tables)]
-    (j/execute! db [(str "DELETE FROM " t)])))
-
-(defn database-fixtures!
-  ([] (database-fixtures! [:posts :users :comments]))
-  ([tables]
-     (fn [f]
-       (clean-tables! tables)
-       (f)
-       (clean-tables! tables)))) ; do it after as well in case a test breaks
-
-(defn rand-str
-  [l]
-  (dg/string #(rand-nth (seq "abcdefghijklmnopqrstuwvxyz ")) l))
-
-(defn view-query
-  [view]
-  (j/query db (hsql/format view)))
-
-(defn view-action!
-  [action]
-  (j/execute! db (hsql/format action)))
-
-(defn user-fixture!
-  [name]
-  (view-action! (hsql/build :insert-into :users :values [{:name name :created_on (sql-ts)}])))
-
-(def user-fixture (atom nil))
-
-(defn with-user-fixture!
-  ([f] (with-user-fixture! "test user" f))
-  ([name f]
-     (user-fixture! name)
-     (let [user (first (j/query db ["SELECT * FROM users WHERE name = ?" name]))]
-       (reset! user-fixture user)
-       (f)
-       (reset! user-fixture nil))))
-
-(defn gen-n-users!
-  [n]
-  (dotimes [n n] (user-fixture! (rand-str 10)))
-  (j/query db ["SELECT * FROM users"]))
-
-(defn insert-post-tmpl
-  [uid title body]
-  (hsql/build :insert-into :posts :values [{:user_id uid :title title :body body :created_on (sql-ts)}]))
-
-(defn post-fixture!
-  [uid title body]
-  (view-action! (insert-post-tmpl uid title body)))
-
-(defn gen-n-posts-for-user!
-  [n uid]
-  (dotimes [n n] (post-fixture! uid (rand-str 20) (rand-str 100))))
-
-(defn users-tmpl
-  []
-  (hsql/build :select [:id :name :created_on] :from :users))
-
-(defn user-posts-tmpl
-  [user_id]
-  (hsql/build :select [:u.id :u.name :p.title :p.body :p.created_on]
-              :from {:posts :p}
-              :join [[:users :u][:= :u.id :p.user_id]]
-              :where [:= :p.user_id user_id]))
-
-(defn users-posts-tmpl
-  []
-  (hsql/build :select [[:u.id :user_id] :u.name :p.id :p.title :p.body :p.created_on]
-              :from {:users :u}
-              :left-join [[:posts :p][:= :u.id :p.user_id]]))
-
-(defn all-comments-tmpl
-  []
-  (hsql/build :select [:id :body :created_on] :from {:comments :c}))
-
-(def templates
-  {:users        {:fn #'users-tmpl}
-   :user-posts   {:fn #'user-posts-tmpl}
-   :all-comments {:fn #'all-comments-tmpl}})

test/views/hint_tests.clj

@@ -0,0 +1,275 @@
+(ns views.hint-tests
+  (:use
+    clojure.test
+    views.test-helpers
+    views.protocols
+    views.core
+    views.test-view-system)
+  (:import (clojure.lang Atom)))
+
+
+(def test-sent-data
+  (atom []))
+
+(defn test-send-fn [subscriber-key [view-sig view-data]]
+  (swap! test-sent-data conj {:subscriber-key subscriber-key
+                              :view-sig       view-sig
+                              :view-data      view-data}))
+
+(def test-options (merge default-options
+                         {:views views
+                          :send-fn test-send-fn}))
+
+(defn clear-sent-data-fixture [f]
+  (reset! test-sent-data [])
+  (f))
+
+(use-fixtures :each clear-sent-data-fixture reset-test-views-system reset-memory-db-fixture)
+
+
+
+;; tests
+
+(deftest refresh-views!-instantly-attempts-view-refresh-with-given-hints
+  (let [options         test-options
+        hints-refreshed (atom [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; with a view subscription (any subscription will do)
+    (with-redefs [subscribed-views (fn [_] #{(->view-sig :namespace :fake-view [])})
+                  refresh-view!    (fn [_ hints _] (swap! hints-refreshed into hints))]
+      ; 2. trigger refresh by calling refresh-views! with hints
+      (refresh-views! test-views-system [(hint :namespace [:foo] :fake-type)])
+      (is (contains-only? @hints-refreshed
+                          [(hint :namespace [:foo] :fake-type)]))
+      (reset! hints-refreshed [])
+      ; 3. same thing again, but passing in multiple hints
+      (refresh-views! test-views-system
+                      [(hint :namespace [:foo] :fake-type)
+                       (hint :namespace [:bar] :fake-type)])
+      (is (contains-only? @hints-refreshed
+                          [(hint :namespace [:foo] :fake-type)
+                           (hint :namespace [:bar] :fake-type)]))
+      (reset! hints-refreshed []))
+    ; now, without any view subscriptions
+    (with-redefs [subscribed-views (fn [_] #{})
+                  refresh-view!    (fn [_ hints _] (swap! hints-refreshed into hints))]
+      ; 4. again trigger refresh by calling refresh-views! with hints
+      (refresh-views! test-views-system [(hint :namespace [:foo] :fake-type)])
+      (is (empty? @hints-refreshed))
+      (reset! hints-refreshed []))))
+
+(deftest refresh-watcher-runs-at-specified-interval-and-picks-up-queued-hints
+  (let [options         test-options
+        hints-refreshed (atom [])]
+    (with-redefs [subscribed-views (fn [_] #{(->view-sig :namespace :fake-view [])})
+                  refresh-view!    (fn [_ hints _] (swap! hints-refreshed into hints))]
+      ; 1. init views
+      (init! test-views-system options)
+      ; 2. queue a hint and wait until the next refresh interval
+      (queue-hints! test-views-system [(hint :namespace [:foo] :fake-type)])
+      (wait-for-refresh-interval options)
+      (is (contains-only? @hints-refreshed
+                          [(hint :namespace [:foo] :fake-type)]))
+      (reset! hints-refreshed [])
+      ; 3. queue multiple hints and wait again
+      (queue-hints! test-views-system
+                    [(hint :namespace [:foo] :fake-type)
+                     (hint :namespace [:bar] :fake-type)])
+      (wait-for-refresh-interval options)
+      (is (contains-only? @hints-refreshed
+                          [(hint :namespace [:foo] :fake-type)
+                           (hint :namespace [:bar] :fake-type)]))
+      (reset! hints-refreshed [])
+      ; 4. queue up no hints and wait
+      (wait-for-refresh-interval options)
+      (reset! hints-refreshed []))))
+
+(deftest refresh-worker-thread-processes-relevant-hints
+  (let [options         test-options
+        views-refreshed (atom [])]
+    ; 1. init views
+    (init! test-views-system options)
+    (with-redefs [subscribed-views (fn [_] #{(->view-sig :a :foo [])})
+                  do-view-refresh! (fn [_ view-sig] (swap! views-refreshed into [view-sig]))]
+      ; 2. trigger refresh by calling refresh-views! with relevant hint
+      (refresh-views! test-views-system [(hint :a [:foo] memory-view-hint-type)])
+      (wait-for-refresh-views)
+      (is (contains-only? @views-refreshed [(->view-sig :a :foo [])]))
+      (reset! views-refreshed [])
+      ; 3. same thing again, but passing in multiple hints (1 relevant, 1 not)
+      (refresh-views! test-views-system [(hint :a [:foo] memory-view-hint-type)
+                       (hint :a [:bar] memory-view-hint-type)])
+      (wait-for-refresh-views)
+      (is (contains-only? @views-refreshed [(->view-sig :a :foo [])]))
+      (reset! views-refreshed [])
+      ; 4. and lastly, passing in only irrelevant hints
+      (refresh-views! test-views-system
+                      [(hint :b [:foo] memory-view-hint-type)
+                       (hint :a [:foo] :some-other-type)])
+      (wait-for-refresh-views)
+      (is (empty? @views-refreshed))
+      (reset! views-refreshed []))))
+
+; this test is really just testing that our helper function memory-db-assoc-in! works as we expect it to
+; (otherwise, it is entirely redundant given the above tests)
+(deftest test-memory-db-operation-triggers-proper-refresh-hints
+  (let [options         test-options
+        hints-refreshed (atom [])
+        views-refreshed (atom [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; first tests verifying that correct hints are being sent out (don't care if relevant or not yet)
+    (with-redefs [subscribed-views (fn [_] #{(->view-sig :a :foo [])})
+                  refresh-view!    (fn [_ hints _] (swap! hints-refreshed into hints))]
+      (memory-db-assoc-in! test-views-system :a [:foo] 42)
+      (memory-db-assoc-in! test-views-system :a [:bar] 3.14)
+      (memory-db-assoc-in! test-views-system :b [:baz] [10 20 30])
+      (wait-for-refresh-views)
+      (is (contains-only? @hints-refreshed
+                          [(hint :a [:foo] memory-view-hint-type)
+                           (hint :a [:bar] memory-view-hint-type)
+                           (hint :b [:baz] memory-view-hint-type)]))
+      (reset! views-refreshed []))
+    ; now we test that relevant views were recognized as relevant and forwarded on to be used to
+    ; trigger actual refreshes of view data
+    (with-redefs [subscribed-views (fn [_] #{(->view-sig :a :foo [])})
+                  do-view-refresh! (fn [_ view-sig] (swap! views-refreshed into [view-sig]))]
+      ; 2. update memory database (in a location covered by the subscribed view)
+      (memory-db-assoc-in! test-views-system :a [:foo] 1337)
+      (wait-for-refresh-interval options)
+      (is (contains-only? @views-refreshed [(->view-sig :a :foo [])]))
+      (reset! views-refreshed [])
+      ; 3. same thing again, but update a different location not covered by any subscription
+      (memory-db-assoc-in! test-views-system :a [:bar] 1234.5678)
+      (wait-for-refresh-interval options)
+      (is (empty? @views-refreshed)))))
+
+(deftest relevant-hints-cause-refreshed-data-to-be-sent-to-subscriber
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [original-view-data (get-view-data test-views-system view-sig)
+          updated-view-data  21
+          subscribe-result   (subscribe! test-views-system view-sig subscriber-key nil)]
+      ; 3. block until subscription finishes. we don't care about the initial view data refresh
+      (while (not (realized? subscribe-result)))
+      (reset! test-sent-data [])
+      (is (= (hash original-view-data) (get-in @test-views-system [:hashes view-sig])))
+      ; 4. change some test data that is covered by the view subscription
+      (memory-db-assoc-in! test-views-system :a [:foo] updated-view-data)
+      (wait-for-refresh-views)
+      (is (= (hash updated-view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      updated-view-data}])))))
+
+(deftest irrelevant-hints-dont-trigger-refreshes
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key nil)]
+      ; 3. block until subscription finishes. we don't care about the initial view data refresh
+      (while (not (realized? subscribe-result)))
+      (reset! test-sent-data [])
+      ; 4. change some test data that is NOT covered by the view subscription
+      (memory-db-assoc-in! test-views-system :b [:foo] 6)
+      (memory-db-assoc-in! test-views-system :a [:bar] 7)
+      (wait-for-refresh-views)
+      (is (empty? @test-sent-data)))))
+
+(deftest refreshes-not-sent-if-view-data-is-unchanged-since-last-refresh
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [updated-view-data 1111
+          subscribe-result  (subscribe! test-views-system view-sig subscriber-key nil)]
+      ; 3. block until subscription finishes. we don't care about the initial view data refresh
+      (while (not (realized? subscribe-result)))
+      (reset! test-sent-data [])
+      ; 4. change some test data, will cause a refresh to be sent out
+      (memory-db-assoc-in! test-views-system :a [:foo] updated-view-data)
+      (wait-for-refresh-views)
+      (is (= (hash updated-view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      updated-view-data}]))
+      (reset! test-sent-data [])
+      ; 5. manually trigger another refresh for the view
+      (refresh-views! test-views-system [(hint :a [:foo] memory-view-hint-type)])
+      (wait-for-refresh-views)
+      (is (empty? @test-sent-data))
+      ; 6. also try "updating" the db with the same values
+      (memory-db-assoc-in! test-views-system :a [:foo] updated-view-data)
+      (wait-for-refresh-views)
+      (is (empty? @test-sent-data)))))
+
+(deftest refresh-queue-drops-duplicate-hints
+  (let [options        (-> test-options
+                           ; enable statistics collection
+                           (assoc :stats-log-interval 10000))
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. prematurely stop refresh worker threads so that we can more easily inspect the
+    ;    internal refresh queue's entries. the refresh worker threads are what remove
+    ;    hints from the refresh queue as they are added to it.
+    (stop-refresh-worker-threads test-views-system)
+    ; 3. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key nil)]
+      ; 4. block until subscription finishes
+      (while (not (realized? subscribe-result)))
+      (is (= 0 (get-in @test-views-system [:statistics :deduplicated])))
+      ; 5. add duplicate hints by changing the same set of data twice
+      ;    (hints will stay in the queue forever because we stopped the worker threads)
+      (memory-db-assoc-in! test-views-system :a [:foo] 6)
+      (memory-db-assoc-in! test-views-system :a [:foo] 7)
+      (wait-for-refresh-views)
+      (is (= 1 (get-in @test-views-system [:statistics :deduplicated])))
+      (is (= [view-sig]
+             (vec (:refresh-queue @test-views-system)))))))
+
+(deftest refresh-queue-drops-hints-when-full
+  (let [options        (-> test-options
+                           ; enable statistics collection
+                           (assoc :stats-log-interval 10000
+                                  :refresh-queue-size 1))
+        subscriber-key 123
+        view-sig-a     (->view-sig :a :foo [])
+        view-sig-b     (->view-sig :b :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. prematurely stop refresh worker threads so that we can more easily inspect the
+    ;    internal refresh queue's entries. the refresh worker threads are what remove
+    ;    hints from the refresh queue as they are added to it.
+    (stop-refresh-worker-threads test-views-system)
+    ; 3. subscribe to a view
+    ; note: log* redef is to suppress error log output which will normally happen whenever
+    ;       another item is added to the refresh queue when it's already full
+    (with-redefs [clojure.tools.logging/log* (fn [& _])]
+      (let [subscribe-a (subscribe! test-views-system view-sig-a subscriber-key nil)
+            subscribe-b (subscribe! test-views-system view-sig-b subscriber-key nil)]
+        ; 4. block until subscription finishes
+        (while (or (not (realized? subscribe-a))
+                   (not (realized? subscribe-b))))
+        (is (= 0 (get-in @test-views-system [:statistics :dropped])))
+        ; 5. change some data affecting the subscribed view, resulting in more then 1 hint
+        ;    being added to the refresh queue
+        (memory-db-assoc-in! test-views-system :a [:foo] 101010)
+        (memory-db-assoc-in! test-views-system :b [:foo] 010101)
+        (wait-for-refresh-views)
+        (is (= 1 (get-in @test-views-system [:statistics :dropped])))
+        (is (= [view-sig-a]
+               (vec (:refresh-queue @test-views-system))))))))

test/views/persistence/memory_test.clj

@@ -1,61 +0,0 @@
-(ns views.persistence.memory-test
-  (:require
-    [views.persistence.core :refer :all]
-    [views.persistence.memory :refer [new-memory-persistence]]
-    [views.fixtures :as vf]
-    [clojure.test :refer [use-fixtures deftest is run-all-tests]]))
-
-(deftest memory-persistence
-  (let [p  (new-memory-persistence)
-        vd (subscribe! p vf/templates :ns [:users] 1)]
-    ;; This sort of test isn't great as it depends on the internal
-    ;; structure unrlated to memory persistence.
-    (is (= vd
-           {:view-sig [:users], :view {:from [:users], :select [:id :name :created_on]}, :refresh-only? nil}))
-
-    ;; Ensure that we are subscribed.
-    (is (= (subscriptions p :ns [[:users]])
-           {[:users] #{1}}))
-
-    ;; Subsequent calls return same vd.
-    (is (= (subscribe! p vf/templates :ns [:users] 3)
-           vd))
-
-    ;; And subscription is correct.
-    (is (= (subscriptions p :ns [[:users]])
-           {[:users] #{1 3}}))
-
-    ;; Missing subscription returns nothing.
-    (is (= (subscriptions p :ns [[:missing]])
-           {}))
-
-    ;; Duplicate subscription is ignored.
-    (subscribe! p vf/templates :ns [:users] 3)
-    (is (= (subscriptions p :ns [[:users]])
-           {[:users] #{1 3}}))
-
-    ;; We can subscribe to multiple views.
-    (subscribe! p vf/templates :ns [:user-posts 1] 5)
-    (is (= (subscriptions p :ns [[:users] [:user-posts 1]])
-           {[:users]      #{1 3}
-            [:user-posts 1] #{5}}))
-
-    ;; Can we unsubscribe a view.
-    (unsubscribe! p :ns [:users] 3)
-    (is (= (subscriptions p :ns [[:users]])
-           {[:users] #{1}}))
-
-    ;; Remove last item in a view makes it go away.
-    (unsubscribe! p :ns [:users] 1)
-    (is (= (subscriptions p :ns [[:users]])
-           {}))
-    (is (= (map :view-sig (view-data p :ns :users))
-           [[:user-posts 1]]))
-
-    ;; Unsubscribe all works.
-    (subscribe! p vf/templates :ns [:users] 7)
-    (subscribe! p vf/templates :ns [:users] 5)
-    (unsubscribe-all! p :ns 5)
-    (is (= (subscriptions p :ns [[:users] [:user-posts 1]])
-           {[:users] #{7}}))))
-

test/views/repl.clj

@@ -1,37 +0,0 @@
-(ns views.repl
-  (:require
-   [honeysql.core :as hsql]
-   [edl.core :refer [defschema]]
-   [views.core :as vc]
-   [views.subscribed-views :as sv]
-   [views.fixtures :as vf]
-   [clojure.data.generators :as dg]
-   [views.db.core :as vdb]
-   [clj-logging-config.log4j :refer [set-logger! set-loggers!]]))
-
-(defn rand-str
-  ([] (rand-str 10))
-  ([n] (dg/string #(rand-nth (clojure.string/split "abcdefghijklmnopqrstuvwxyz" #"\B")) n)))
-
-(defschema test-schema vf/db "public")
-
-(def user-insert (hsql/build :insert-into :users :values [{:name (rand-str) :created_on (vf/sql-ts)}]))
-
-(defn make-config
-  ([] (make-config vf/templates))
-  ([templates] (vc/config {:db vf/db :schema test-schema :templates templates :unsafe? true})))
-
-(defn test-subscribe
-  ([sk views] (test-subscribe sk views (make-config)))
-  ([sk views opts]
-     (sv/subscribe-views (:base-subscribed-views opts) {:subscriber-key sk :views [[:users]]})))
-
-(comment
-  (require '[clj-logging-config.log4j :as lc] '[views.repl :as vr] '[views.db.core :as vdb] :reload)
-  (lc/set-loggers! "views.base-subscribed-views" {:level :info})
-  (def conf (vr/make-config))
-  (vr/test-subscribe 1 [[:users]])
-  (vdb/vexec! conf vr/user-insert)
-  (vr/test-subscribe 2 [[:users]])
-  (vdb/vexec! conf vr/user-insert)
-  )

test/views/subscription_tests.clj

@@ -0,0 +1,407 @@
+(ns views.subscription-tests
+  (:use
+    clojure.test
+    views.test-helpers
+    views.protocols
+    views.core
+    views.test-view-system))
+
+
+(def test-sent-data
+  (atom []))
+
+(defn test-send-fn [subscriber-key [view-sig view-data]]
+  (swap! test-sent-data conj {:subscriber-key subscriber-key
+                              :view-sig       view-sig
+                              :view-data      view-data}))
+
+(def test-options (merge default-options
+                         {:views views
+                          :send-fn test-send-fn}))
+
+(defn clear-sent-data-fixture [f]
+  (reset! test-sent-data [])
+  (f))
+
+(use-fixtures :each clear-sent-data-fixture reset-test-views-system reset-memory-db-fixture)
+
+
+
+;; tests
+
+(deftest can-subscribe-to-a-view
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :namespace :foo [])
+        context        {:my-data "arbitrary application context data"}]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      (is (future? subscribe-result))
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig])))
+      ; 3. block until subscription finishes (data retrieval + initial view refresh)
+      ;    (in this particular unit test, there is really no point in waiting)
+      (while (not (realized? subscribe-result)))
+      (is (= #{view-sig} (subscribed-views test-views-system))))))
+
+(deftest subscribing-results-in-initial-view-data-being-sent
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        context        {:my-data "arbitrary application context data"}]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      ; 3. block until subscription finishes (data retrieval + initial view refresh)
+      (while (not (realized? subscribe-result)))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}])))))
+
+(deftest can-unsubscribe-from-a-view
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        context        {:my-data "arbitrary application context data"}]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      ; 3. block until subscription finishes
+      (while (not (realized? subscribe-result)))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      ; 4. unsubscribe
+      (unsubscribe! test-views-system view-sig subscriber-key context)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest multiple-subscription-and-unsubscriptions
+  (let [options          test-options
+        subscriber-key-a 123
+        subscriber-key-b 456
+        view-sig         (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          subscribe-a      (subscribe! test-views-system view-sig subscriber-key-a nil)
+          subscribe-b      (subscribe! test-views-system view-sig subscriber-key-b nil)]
+      ; 3. block until both subscriptions finish
+      (while (or (not (realized? subscribe-a))
+                 (not (realized? subscribe-b))))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= [subscriber-key-a subscriber-key-b] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key-a])))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key-b])))
+      (is (= #{subscriber-key-a subscriber-key-b} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key-a
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}
+                           {:subscriber-key subscriber-key-b
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}]))
+      ; 4. have one of the subscribers unsubscribe
+      (unsubscribe! test-views-system view-sig subscriber-key-a nil)
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= [subscriber-key-b] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key-b])))
+      (is (= #{subscriber-key-b} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      ; 5. have the last subscriber also unsubscribe
+      (unsubscribe! test-views-system view-sig subscriber-key-b nil)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest subscriptions-to-different-views
+  (let [options          test-options
+        subscriber-key-a 123
+        subscriber-key-b 456
+        view-sig-a       (->view-sig :a :foo [])
+        view-sig-b       (->view-sig :a :bar [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data-a (get-view-data test-views-system view-sig-a)
+          view-data-b (get-view-data test-views-system view-sig-b)
+          subscribe-a (subscribe! test-views-system view-sig-a subscriber-key-a nil)
+          subscribe-b (subscribe! test-views-system view-sig-b subscriber-key-b nil)]
+      ; 3. block until both subscriptions finish
+      (while (or (not (realized? subscribe-a))
+                 (not (realized? subscribe-b))))
+      (is (= #{view-sig-a view-sig-b} (subscribed-views test-views-system)))
+      (is (= [subscriber-key-a subscriber-key-b] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig-a} (get-in @test-views-system [:subscribed subscriber-key-a])))
+      (is (= #{view-sig-b} (get-in @test-views-system [:subscribed subscriber-key-b])))
+      (is (= #{subscriber-key-a} (get-in @test-views-system [:subscribers view-sig-a])))
+      (is (= #{subscriber-key-b} (get-in @test-views-system [:subscribers view-sig-b])))
+      (is (= (hash view-data-a) (get-in @test-views-system [:hashes view-sig-a])))
+      (is (= (hash view-data-b) (get-in @test-views-system [:hashes view-sig-b])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key-a
+                            :view-sig       (dissoc view-sig-a :namespace)
+                            :view-data      view-data-a}
+                           {:subscriber-key subscriber-key-b
+                            :view-sig       (dissoc view-sig-b :namespace)
+                            :view-data      view-data-b}]))
+      ; 4. have one of the subscribers unsubscribe
+      (unsubscribe! test-views-system view-sig-a subscriber-key-a nil)
+      (is (= #{view-sig-b} (subscribed-views test-views-system)))
+      (is (= [subscriber-key-b] (keys (:subscribed @test-views-system))))
+      (is (empty? (get-in @test-views-system [:subscribed subscriber-key-a])))
+      (is (= #{view-sig-b} (get-in @test-views-system [:subscribed subscriber-key-b])))
+      (is (= #{subscriber-key-b} (get-in @test-views-system [:subscribers view-sig-b])))
+      (is (empty? (get-in @test-views-system [:subscribers view-sig-a])))
+      (is (empty? (get-in @test-views-system [:hashes view-sig-a])))
+      (is (= (hash view-data-b) (get-in @test-views-system [:hashes view-sig-b])))
+      ; 5. have the last subscriber also unsubscribe
+      (unsubscribe! test-views-system view-sig-b subscriber-key-b nil)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest duplicate-subscriptions-do-not-cause-problems
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :a :foo [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          first-subscribe  (subscribe! test-views-system view-sig subscriber-key nil)
+          second-subscribe (subscribe! test-views-system view-sig subscriber-key nil)]
+      ; 3. block until both subscriptions finish
+      (while (or (not (realized? first-subscribe))
+                 (not (realized? second-subscribe))))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}
+                           {:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}]))
+      ; 4. unsubscribe. only need to do this once, since only one subscription
+      ;    should exist in the view system
+      (unsubscribe! test-views-system view-sig subscriber-key nil)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest subscribing-to-non-existant-view-raises-exception
+  (let [options        test-options
+        subscriber-key 123
+        view-sig       (->view-sig :namespace :non-existant-view [])]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (is (thrown? Exception (subscribe! test-views-system view-sig subscriber-key nil)))))
+
+(deftest subscribe-and-unsubscribe-use-namespace-fn-if-set-and-no-namespace-in-view-sig
+  (let [subscriber-key 123
+        view-sig       (->view-sig :foo [])
+        context        "some arbitrary context data"
+        namespace-fn   (fn [view-sig* subscriber-key* context*]
+                         (is (= view-sig view-sig*))
+                         (is (= subscriber-key subscriber-key*))
+                         (is (= context context*))
+                         :b)
+        options        (-> test-options
+                           (assoc :namespace-fn namespace-fn))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [; with the above namespace-fn, subscribe will internally use this view sig
+          ; when setting up subscription info within view-system. application code
+          ; shouldn't need to worry about this, but we will in this unit test
+          view-sig-with-ns (->view-sig :b :foo [])
+          ; such as right here, we need to use the actual namespace that was set in
+          ; view-system to pass in the same parameters that subscribe! will use for
+          ; the view during it's initial view data refresh
+          view-data        (get-view-data test-views-system view-sig-with-ns)
+          ; passing in view-sig *without* namespace
+          subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      ; 3. block until subscription finishes
+      (while (not (realized? subscribe-result)))
+      (is (= #{view-sig-with-ns} (subscribed-views test-views-system)))
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig-with-ns} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig-with-ns])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig-with-ns])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}]))
+      ; 4. unsubscribe.
+      ; NOTE: we are passing in view-sig, not view-sig-with-ns. this is because
+      ;       proper namespace-fn's should be consistent with what namespace they
+      ;       return given the same inputs. ideal namespace-fn implementations will
+      ;       also keep this in mind even if context could vary between subscribe!
+      ;       and unsubscribe! calls.
+      (unsubscribe! test-views-system view-sig subscriber-key context)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest subscribe-and-unsubscribe-do-not-use-namespace-fn-if-namespace-included-in-view-sig
+  (let [subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        context        "some arbitrary context data"
+        namespace-fn   (fn [view-sig* subscriber-key* context*]
+                         ; if this function is used, it will mess up several assertions in this unit test
+                         :b)
+        options        (-> test-options
+                           (assoc :namespace-fn namespace-fn))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      ; 3. block until subscription finishes
+      (while (not (realized? subscribe-result)))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}]))
+      ; 4. unsubscribe.
+      (unsubscribe! test-views-system view-sig subscriber-key context)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest unauthorized-subscription-using-auth-fn
+  (let [subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        context        "some arbitrary context data"
+        auth-fn        (fn [view-sig* subscriber-key* context*]
+                         (is (= view-sig view-sig*))
+                         (is (= subscriber-key subscriber-key*))
+                         (is (= context context*))
+                         ; false = unauthorized
+                         false)
+        options        (-> test-options
+                           (assoc :auth-fn auth-fn))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      (is (nil? subscribe-result))
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest unauthorized-subscription-using-auth-fn-calls-on-unauth-fn-when-set
+  (let [subscriber-key    123
+        view-sig          (->view-sig :a :foo [])
+        context           "some arbitrary context data"
+        auth-fn           (fn [view-sig* subscriber-key* context*]
+                            (is (= view-sig view-sig*))
+                            (is (= subscriber-key subscriber-key*))
+                            (is (= context context*))
+                            ; false = unauthorized
+                            false)
+        on-unauth-called? (atom false)
+        on-unauth-fn      (fn [view-sig* subscriber-key* context*]
+                            (is (= view-sig view-sig*))
+                            (is (= subscriber-key subscriber-key*))
+                            (is (= context context*))
+                            (reset! on-unauth-called? true))
+        options           (-> test-options
+                              (assoc :auth-fn auth-fn
+                                     :on-unauth-fn on-unauth-fn))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      (is (nil? subscribe-result))
+      (is @on-unauth-called?)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system))))))
+
+(deftest authorized-subscription-using-auth-fn
+  (let [subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        context        "some arbitrary context data"
+        auth-fn        (fn [view-sig* subscriber-key* context*]
+                         (is (= view-sig view-sig*))
+                         (is (= subscriber-key subscriber-key*))
+                         (is (= context context*))
+                         true)
+        options        (-> test-options
+                           (assoc :auth-fn auth-fn))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [view-data        (get-view-data test-views-system view-sig)
+          subscribe-result (subscribe! test-views-system view-sig subscriber-key context)]
+      (while (not (realized? subscribe-result)))
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (= [subscriber-key] (keys (:subscribed @test-views-system))))
+      (is (= #{view-sig} (get-in @test-views-system [:subscribed subscriber-key])))
+      (is (= #{subscriber-key} (get-in @test-views-system [:subscribers view-sig])))
+      (is (= (hash view-data) (get-in @test-views-system [:hashes view-sig])))
+      (is (contains-only? @test-sent-data
+                          [{:subscriber-key subscriber-key
+                            :view-sig       (dissoc view-sig :namespace)
+                            :view-data      view-data}])))))
+
+(deftest unsubscribe-before-subscription-finishes-does-not-result-in-stuck-view
+  (let [subscriber-key 123
+        view-sig       (->view-sig :a :foo [])
+        options        (-> test-options
+                           (assoc :views slow-views))]
+    ; 1. init views
+    (init! test-views-system options)
+    ; 2. subscribe to a view
+    (let [subscribe-result (subscribe! test-views-system view-sig subscriber-key nil)]
+      (is (= #{view-sig} (subscribed-views test-views-system)))
+      (is (not (realized? subscribe-result)))
+      ; 3. unsubscribe before subscription finishes (still waiting on initial data
+      ;    retrieval to finish)
+      (unsubscribe! test-views-system view-sig subscriber-key nil)
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system)))
+      (is (empty? @test-sent-data))
+      ; 4. wait for subscription to finish finally
+      (while (not (realized? subscribe-result)))
+      (is (empty? (keys (:subscribed @test-views-system))))
+      (is (empty? (keys (:subscribers @test-views-system))))
+      (is (empty? (subscribed-views test-views-system)))
+      (is (empty? (:hashes @test-views-system)))
+      (is (empty? @test-sent-data)))))

test/views/test_db.sql

@@ -1,18 +0,0 @@
-CREATE ROLE views_user LOGIN PASSWORD 'password';
-CREATE DATABASE views_test OWNER views_user;
-\c postgresql://localhost/views_test;
-CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL, created_on TIMESTAMP NOT NULL);
-CREATE TABLE posts (id SERIAL PRIMARY KEY,
-                    title TEXT NOT NULL,
-                    body TEXT NOT NULL,
-                    created_on TIMESTAMP NOT NULL,
-                    user_id INTEGER NOT NULL,
-                    FOREIGN KEY (user_id) REFERENCES users(id));
-CREATE TABLE comments (id SERIAL PRIMARY KEY,
-                       body TEXT NOT NULL,
-                       created_on TIMESTAMP NOT NULL,
-                       post_id INTEGER NOT NULL,
-                       FOREIGN KEY (post_id) REFERENCES posts(id));
-ALTER TABLE users OWNER TO views_user;
-ALTER TABLE posts OWNER TO views_user;
-ALTER TABLE comments OWNER TO views_user;

test/views/test_helpers.clj

@@ -0,0 +1,55 @@
+(ns views.test-helpers
+  (:use
+    clojure.test
+    views.protocols
+    views.core)
+  (:import (clojure.lang Atom)))
+
+(defn contains-view?
+  [^Atom view-system view-id]
+  (let [view (get (:views @view-system) view-id)]
+    (and view
+         (satisfies? IView view))))
+
+; the purpose of this function is to compare collections when the order of the elements
+; is not important, but there could be duplicates (so a set is not being used). some of
+; the operations we test are asynchronous with multiple threads performing the same
+; operation simultaneously, so at times our tests that record these operations being done
+; could end up with collections of items that are out of order between test runs.
+; this is not a problem or bug in views, but just a consequence of the multithreaded
+; operation of the library.
+(defn contains-only?
+  [coll elements]
+  (and (= (count coll)
+          (count elements))
+       (every?
+         #(boolean (some #{%} elements))
+         coll)
+       (every?
+         #(boolean (some #{%} coll))
+         elements)))
+
+(defn get-view-data
+  [^Atom view-system view-sig]
+  (data (get-in @view-system [:views (:view-id view-sig)])
+        (:namespace view-sig)
+        (:parameters view-sig)))
+
+; the 200 being used is just a number i pulled out of thin air that "felt good"
+
+(defn wait-for-refresh-views []
+  (Thread/sleep 200))
+
+(defn wait-for-refresh-interval [options]
+  (Thread/sleep (+ 200 (:refresh-interval options))))
+
+; this is kind of a hack, but necessary for some tests where we want to inspect
+; the items being sent to the refresh queue without worker threads picking out
+; the added items almost instantly.
+(defn stop-refresh-worker-threads
+  [^Atom view-system]
+  (swap! view-system assoc :stop-workers? true)
+  (doseq [^Thread t (:workers @view-system)]
+    (.interrupt t)
+    (.join t))
+  (swap! view-system assoc :workers nil))

test/views/test_view_system.clj

@@ -0,0 +1,71 @@
+(ns views.test-view-system
+  (:use
+    views.protocols
+    views.core)
+  (:import (clojure.lang Atom)))
+
+(def base-memory-db-contents
+  {:a {:foo 1 :bar 200 :baz [1 2 3]}
+   :b {:foo 2 :bar 300 :baz [2 3 4]}})
+
+(def memory-database
+  (atom base-memory-db-contents))
+
+(def test-views-system
+  (atom {}))
+
+(defn reset-memory-db-fixture [f]
+  (reset! memory-database base-memory-db-contents)
+  (f))
+
+(defn reset-test-views-system [f]
+  (reset! test-views-system {})
+  (f)
+  (if (seq @test-views-system)
+    (shutdown! test-views-system)))
+
+(def memory-view-hint-type :memory-db)
+
+(defrecord MemoryView [id ks]
+  IView
+  (id [_] id)
+  (data [_ namespace parameters]
+    (get-in @memory-database (-> [namespace]
+                                 (into ks)
+                                 (into parameters))))
+  (relevant? [_ namespace parameters hints]
+    (some #(and (= namespace (:namespace %))
+                (= ks (:hint %))
+                (= memory-view-hint-type (:type %)))
+          hints)))
+
+(defrecord SlowMemoryView [id ks]
+  IView
+  (id [_] id)
+  (data [_ namespace parameters]
+    ; simulate a slow database query
+    (Thread/sleep 1000)
+    (get-in @memory-database (-> [namespace]
+                                 (into ks)
+                                 (into parameters))))
+  (relevant? [_ namespace parameters hints]
+    (some #(and (= namespace (:namespace %))
+                (= ks (:hint %))
+                (= memory-view-hint-type (:type %)))
+          hints)))
+
+(def views
+  [(MemoryView. :foo [:foo])
+   (MemoryView. :bar [:bar])
+   (MemoryView. :baz [:baz])])
+
+(def slow-views
+  [(SlowMemoryView. :foo [:foo])
+   (SlowMemoryView. :bar [:bar])
+   (SlowMemoryView. :baz [:baz])])
+
+(defn memory-db-assoc-in!
+  [^Atom view-system namespace ks v]
+  (let [ms (swap! memory-database assoc-in (into [namespace] ks) v)]
+    (put-hints! view-system [(hint namespace ks memory-view-hint-type)])
+    ms))