gered/mtgcoll
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

bump versions of things, switch from embedded to user-provided config

Browse source 
"embedded" as in, included in built uberjars. config must now be
specified via the cli using the "--config" option

this necessitated some additional changes to how different components
are instantiated for each of the different run actions (e.g. "web",
"scrape-prices") and also for dev tasks like migrate/rollback. using
mount now to handle component start/stop and dependency management.

some cljs dependencies not updated yet as things like the views
libraries are not yet updated the new sente version
This commit is contained in:
Gered 2018-06-07 19:02:10 -04:00
parent 85e5f104ca
commit 94317bdf7e
17 changed files with 233 additions and 226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -20,3 +20,4 @@ pom.xml.asc
/profiles.clj
/resources/public/cljs
/.repl
/images

19
env/dev/src/user.clj vendored
View file

@ -1,19 +0,0 @@
(ns user
(:use
mtgcoll.core)
(:require
[ragtime.repl :as ragtime]
[mtgcoll.config :as config]
[mtgcoll.db :as db]))
(defn migrate [& args]
(config/load! "config.edn")
(db/setup-config!)
(println "Running migrations on" (:subname @db/db))
(ragtime/migrate (db/get-ragtime-config)))
(defn rollback [& args]
(config/load! "config.edn")
(db/setup-config!)
(println "Rolling back migrations on" (:subname @db/db))
(ragtime/rollback (db/get-ragtime-config) (or (first args) 1)))

27
profiles/dev/src/user.clj Normal file
View file

@ -0,0 +1,27 @@
(ns user
(:use
mtgcoll.core)
(:require
[mount.core :as mount]
[ragtime.repl :as ragtime]
[mtgcoll.cli :as cli]
[mtgcoll.config :as config]
[mtgcoll.db :as db]))
(defn migrate [& args]
(let [{:keys [errors] :as args} (cli/parse-args args)]
(when errors
(cli/show-error! errors)
(System/exit 1))
(mount/start-with-args args #'config/app-config #'db/db)
(println "Running migrations on" (:subname db/db))
(ragtime/migrate (db/get-ragtime-config))))
(defn rollback [& args]
(let [{:keys [errors] :as args} (cli/parse-args args)]
(when errors
(cli/show-error! errors)
(System/exit 1))
(mount/start-with-args args #'config/app-config #'db/db)
(println "Rolling back migrations on" (:subname db/db))
(ragtime/rollback (db/get-ragtime-config) (or (first args) 1))))

112
project.clj
View file

@ -4,42 +4,43 @@
:license {:name "MIT License"
:url "http://opensource.org/licenses/MIT"}
:dependencies [[org.clojure/clojure "1.8.0"]
[org.clojure/clojurescript "1.8.51"]
[ring "1.4.0"]
[ring/ring-defaults "0.2.0"]
[compojure "1.4.0"]
[org.immutant/web "2.1.4"]
[org.clojure/tools.logging "0.3.1"]
[org.clojure/tools.cli "0.3.5"]
[org.clojure/java.jdbc "0.6.1"]
[org.postgresql/postgresql "9.4.1208.jre7"]
:dependencies [[cheshire "5.6.1"]
[clj-http "3.8.0"]
[cljsjs/chartjs "2.0.1-0"]
[cljsjs/showdown "1.4.2-0"]
[com.taoensso/sente "1.8.1"]
[compojure "1.6.0"]
[enlive "1.1.6"]
[gered/config "0.1"]
[gered/views "1.5"]
[gered/views.sql "0.1"]
[gered/views.reagent "0.1"]
[gered/views.reagent.sente "0.1"]
[gered/views.sql "0.1"]
[gered/webtools "0.1.1"]
[gered/webtools.reagent "0.1.1"]
[hiccup "1.0.5"]
[reagent "0.6.0-alpha2"]
[secretary "1.2.3"]
[clj-http "2.2.0"]
[honeysql "0.7.0"]
[ring-middleware-format "0.7.0"]
[ragtime "0.6.0"]
[enlive "1.1.6"]
[slugger "1.0.1"]
[cheshire "5.6.1"]
[cljsjs/chartjs "2.0.1-0"]
[cljsjs/showdown "1.4.2-0"]
[luminus/ring-ttl-session "0.3.1"]
[mount "0.1.12"]
[org.clojure/clojure "1.8.0"]
[org.clojure/clojurescript "1.8.51"]
[org.clojure/java.jdbc "0.6.1"]
[org.clojure/tools.cli "0.3.5"]
[org.clojure/tools.logging "0.4.1"]
[org.clojure/tools.nrepl "0.2.13"]
[org.immutant/web "2.1.9"]
[org.postgresql/postgresql "9.4.1208.jre7"]
[org.webjars/bootstrap "3.3.6"]
[ragtime "0.6.0"]
[reagent "0.6.0"]
[ring "1.6.3"]
[ring-middleware-format "0.7.0"]
[ring-webjars "0.2.0"]
[ring/ring-defaults "0.3.1" :exclusions [javax.servlet/servlet-api]]
[secretary "1.2.3"]
[slugger "1.0.1"]]
[gered/config "0.1"]]
:plugins [[lein-cljsbuild "1.1.3"]
[lein-figwheel "0.5.4-1"]]
:plugins [[lein-cljsbuild "1.1.7"]]
:main mtgcoll.core
@ -48,35 +49,42 @@
:clean-targets ^{:protect false} [:target-path
[:cljsbuild :builds :main :compiler :output-dir]
[:cljsbuild :builds :main :compiler :output-to]]
:cljsbuild {:builds {:main
{:source-paths ["src"]
:figwheel true
:compiler {:main mtgcoll.client.core
:output-to "resources/public/cljs/app.js"
:output-dir "resources/public/cljs/target"
:asset-path "cljs/target"
:source-map true
:optimizations :none
:pretty-print true}}}}
:profiles {:dev {:source-paths ["env/dev/src"]
:resource-paths ["env/dev/resources"]
:dependencies [[pjstadig/humane-test-output "0.8.0"]]
:cljsbuild {:builds
{:main
{:source-paths ["src"]
:compiler {:asset-path "cljs/target"
:main mtgcoll.client.core
:optimizations :none
:output-dir "resources/public/cljs/target"
:output-to "resources/public/cljs/app.js"
:pretty-print true
:source-map true}}}}
:profiles {:dev {:source-paths ["profiles/dev/src"]
:resource-paths ["profiles/dev/resources"]
:dependencies [[binaryage/devtools "0.9.4"]
[pjstadig/humane-test-output "0.8.0"]]
:injections [(require 'pjstadig.humane-test-output)
(pjstadig.humane-test-output/activate!)]
:cljsbuild {:builds {:main {:source-paths ["env/dev/src"]}}}}
:cljsbuild {:builds
{:main
{:source-paths ["profiles/dev/src"]
:compiler {:preloads [devtools.preload]}}}}}
:uberjar {:env {}
:aot :all
:hooks [leiningen.cljsbuild]
:cljsbuild {:jar true
:figwheel false
:builds {:main
{:compiler ^:replace {:output-to "resources/public/cljs/app.js"
:optimizations :advanced
:pretty-print false}}}}}}
:uberjar {:source-paths ["profiles/uberjar/src"]
:resource-paths ["profiles/uberjar/resources"]
:aot :all
:hooks [leiningen.cljsbuild]
:omit-source true
:cljsbuild {:jar true
:builds {:main
{:compiler ^:replace {:optimizations :advanced
:output-to "resources/public/cljs/app.js"
:pretty-print false}}}}}}
:aliases {"uberjar" ["do" ["clean"] ["uberjar"]]
:aliases {"launch" ["run" "--" "--config" "my-config.edn"]
"uberjar" ["do" ["clean"] ["uberjar"]]
"cljsdev" ["do" ["cljsbuild" "once"] ["cljsbuild" "auto"]]
"migrate" ["run" "-m" "user/migrate"]
"rollback" ["run" "-m" "user/rollback"]})
"migrate" ["run" "-m" "user/migrate" "--" "--config" "my-config.edn"]
"rollback" ["run" "-m" "user/rollback" "--" "--config" "my-config.edn"]})

26
src/mtgcoll/cli.clj
View file

@ -41,22 +41,24 @@
(str "Invalid options/arguments.\n"
(string/join \newline errors)))
(defn- exit
[status msg]
(println msg)
(System/exit status))
(defn show-help!
[{:keys [summary] :as parsed-cli}]
(println (->usage-string summary)))
(defn parse-cli-args
(defn show-error!
[{:keys [errors] :as parsed-cli}]
(println (->error-msg errors)))
(defn parse-args
[args]
(let [{:keys [options arguments errors summary] :as parsed-cli} (cli/parse-opts args cli-options)
arguments (or (seq arguments) ["web"])
action (first arguments)
arguments (rest arguments)
valid-action? (boolean (some #{action} (map first actions)))]
(cond
(:help options) (exit 0 (->usage-string summary))
(not valid-action?) (exit 1 (->usage-string summary))
errors (exit 1 (->error-msg errors)))
{:options options
:action (keyword action)
:arguments arguments}))
{:options options
:action (keyword action)
:arguments arguments
:summary summary
:errors errors
:valid-action? valid-action?}))

20
src/mtgcoll/config.clj
View file

@ -1,14 +1,20 @@
(ns mtgcoll.config
(:refer-clojure :exclude [get])
(:require
[config.core :as cfg]))
[clojure.java.io :as io]
[clojure.tools.logging :as log]
[config.core :as config]
[mount.core :as mount :refer [defstate]]))
(defonce app-config (atom {}))
(defn load!
[config-file]
(reset! app-config (cfg/load (or config-file (System/getenv "config")))))
(defstate app-config
:start (let [config-file (get-in (mount/args) [:options :config])]
(if (and config-file
(.exists (io/file config-file)))
(do
(log/info (str "Loading app config from " config-file))
(config/load config-file {}))
(throw (Exception. "No config file specified or does not exist.")))))
(defn get
[& ks]
(apply cfg/get @app-config ks))
(apply config/get app-config ks))

132
src/mtgcoll/core.clj
View file

@ -2,21 +2,22 @@
(:gen-class)
(:require
[clojure.tools.logging :as log]
[clojure.tools.nrepl.server :as nrepl-server]
[compojure.core :refer [routes GET POST]]
[compojure.route :as route]
[immutant.web :as immutant]
[mount.core :as mount :refer [defstate]]
[ring.middleware.defaults :refer [wrap-defaults site-defaults]]
[ring.middleware.format :refer [wrap-restful-format]]
[ring.middleware.reload :refer [wrap-reload]]
[ring-ttl-session.core :refer [ttl-memory-store]]
[taoensso.sente.server-adapters.immutant :refer [sente-web-server-adapter]]
[mtgcoll.cli :refer [parse-cli-args]]
[mtgcoll.cli :as cli]
[mtgcoll.config :as config]
[mtgcoll.db :as db]
[mtgcoll.models.mtgjson :refer [load-mtgjson-data!]]
[mtgcoll.scrapers.image-assets :refer [download-gatherer-set-images! download-gatherer-symbol-images!]]
[mtgcoll.scrapers.prices :refer [update-prices!]]
[mtgcoll.views.core :as views]
[mtgcoll.views.sente :as sente]
[mtgcoll.routes.main-page :refer [main-page-routes]]
[mtgcoll.routes.images :refer [image-routes]]
@ -24,29 +25,6 @@
[mtgcoll.routes.lists :refer [list-routes]]
[mtgcoll.routes.auth :refer [auth-routes]]))
(defn init
[]
(log/info "Starting up web application ...")
(when (config/get :dev?)
(log/info "Running in development environment."))
(db/setup-config!)
(db/verify-connection)
(sente/init!)
(views/init!)
(log/info "Application init finished."))
(defn shutdown
[]
(log/info "Shutting down ...")
(views/shutdown!)
(sente/shutdown!)
(log/info "Application stopped."))
(def handler
(-> (routes
auth-routes
@ -60,55 +38,71 @@
(sente/wrap-sente "/chsk")
(wrap-defaults (assoc-in site-defaults [:session :store] (ttl-memory-store (* 60 30))))))
(defn start-server!
[]
(init)
(let [options (merge
{:port 8080
:host "localhost"
:path "/"
:virtual-host nil
:dispatch? true}
(config/get :web))]
(if (config/get :dev?)
; why the fuck would anyone assume that if you are running in "development mode" you automatically
; want a new browser window/tab to be opened each time the web server is started? AND not provide an
; option to disable this annoying-as-fuck behaviour.
; i mean, jesus christ, it's obviously _not_ possible that i might already have a browser open that i
; might prefer to keep reusing ... !
(with-redefs [clojure.java.browse/browse-url (fn [_])]
(immutant/run-dmc #'handler options))
(immutant/run #'handler options))))
(defstate ^{:on-reload :noop} http-server
:start (let [options (merge
{:host "localhost"
:port 8080
:path "/"
:virtual-host nil
:dispatch? nil}
(config/get :http))]
(log/info "Starting HTTP server: " options)
(immutant/run (if (config/get :dev?)
(wrap-reload #'handler)
#'handler)
options))
:stop (do
(log/info "Stopping HTTP server")
(immutant/stop http-server)))
(defn stop-server!
[]
(if (immutant/stop)
(shutdown)))
(defstate ^{:on-reload :noop} repl-server
:start (when-let [nrepl-config (config/get :nrepl)]
(let [port (or (:port nrepl-config) 4000)]
(log/info "Starting nREPL server on port: " port)
(nrepl-server/start-server :port port)))
:stop (when repl-server
(log/info "Stopping nREPL server")
(nrepl-server/stop-server repl-server)))
(defn -main
[& args]
(let [{:keys [options action arguments]} (parse-cli-args args)]
(config/load! (:config options))
(db/setup-config!)
(db/verify-connection)
(case action
:web
(let [{:keys [options action arguments errors valid-action?] :as args} (cli/parse-args args)]
(cond
(or (:help options)
(not valid-action?))
(cli/show-help! args)
errors
(do
(start-server!)
(.addShutdownHook (Runtime/getRuntime) (Thread. ^Runnable stop-server!)))
(cli/show-error! errors)
(System/exit 1))
:setup-db
(db/initialize-database!)
:else
(case action
:web
(do
(mount/start-with-args args)
(.addShutdownHook (Runtime/getRuntime) (Thread. ^Runnable mount/stop)))
:load-json
(load-mtgjson-data! (first arguments))
:setup-db
(do
(mount/start-with-args args #'config/app-config, #'db/db)
(db/initialize-database!))
:scrape-prices
(if (seq arguments)
(update-prices! (first arguments))
(update-prices!))
:load-json
(do
(mount/start-with-args args #'config/app-config, #'db/db)
(load-mtgjson-data! (first arguments)))
:scrape-images
(do
(download-gatherer-set-images!)
(download-gatherer-symbol-images!)))))
:scrape-prices
(do
(mount/start-with-args args #'config/app-config, #'db/db)
(if (seq arguments)
(update-prices! (first arguments))
(update-prices!)))
:scrape-images
(do
(mount/start-with-args args #'config/app-config, #'db/db)
(download-gatherer-set-images!)
(download-gatherer-symbol-images!))))))

29
src/mtgcoll/db.clj
View file

@ -1,26 +1,27 @@
(ns mtgcoll.db
(:require
[clojure.java.jdbc :as sql]
[clojure.tools.logging :as log]
[mount.core :refer [defstate]]
[ragtime.jdbc :as jdbc]
[ragtime.repl :as ragtime]
[mtgcoll.config :as config]))
(defonce db (atom nil))
(defn setup-config!
[]
(reset! db {:classname "org.postgresql.Driver"
:subprotocol "postgresql"
:subname (str "//" (config/get :db :host) ":" (or (config/get :db :port) 5432) "/" (config/get :db :name))
:user (config/get :db :username)
:password (config/get :db :password)}))
(defn verify-connection
[]
(sql/query @db "select 1"))
(defstate db
:start (let [db {:classname "org.postgresql.Driver"
:subprotocol "postgresql"
:subname (str "//" (config/get :db :host) ":" (or (config/get :db :port) 5432) "/" (config/get :db :name))
:user (config/get :db :username)
:password (config/get :db :password)}]
(log/info "Using DB: " (:subname db))
(sql/query db "select 1")
db)
:stop (do
(log/info "Stopping DB: " (:subname db))
nil))
(defn get-ragtime-config []
{:datastore (jdbc/sql-database @db)
{:datastore (jdbc/sql-database db)
:migrations (jdbc/load-resources "migrations")})
(defn initialize-database!

10
src/mtgcoll/models/cards.clj
View file

@ -6,9 +6,9 @@
(defn get-card-image-info
[card-id]
(sql/query @db ["select set_code, image_name
from cards
where id = ?" card-id]
(sql/query db ["select set_code, image_name
from cards
where id = ?" card-id]
{:result-set-fn first}))
(defn get-matching-card-ids
@ -26,13 +26,13 @@
[:= :number number]))
(remove nil? x)
(vec x))}]
(seq (sql/query @db (hsql/format q) {:row-fn :id}))))
(seq (sql/query db (hsql/format q) {:row-fn :id}))))
(defn update-price!
[card-id price-source price online?]
;; written assuming postgresql server is _not_ 9.5+ (so, without access to UPSERT functionality)
(sql/with-db-transaction
[dt @db]
[dt db]
(let [num-updates (first
(sql/execute!
dt

2
src/mtgcoll/models/collection.clj
View file

@ -12,7 +12,7 @@
public-only? (nil? user-id)]
(with-view-transaction
view-system
[dt @db]
[dt db]
(if-not (first (jdbc/query dt ["select count(*) from lists where id = ? and is_public in (true, ?)" list-id public-only?]))
(throw (new Exception (str "Not authorized to update list:" list-id)))
(let [num-updates (first

10
src/mtgcoll/models/lists.clj
View file

@ -6,7 +6,7 @@
(defn add-list!
[name public? requires-qualities?]
(let [result (vexec! view-system @db
(let [result (vexec! view-system db
["insert into lists
(name, is_public, require_qualities)
values
@ -17,14 +17,14 @@
(defn remove-list!
[list-id]
(vexec! view-system @db
(vexec! view-system db
["delete from lists
where id = ?"
(int list-id)]))
(defn update-list-name!
[list-id name]
(vexec! view-system @db
(vexec! view-system db
["update lists
set name = ?
where id = ?"
@ -32,7 +32,7 @@
(defn update-list-note!
[list-id note]
(vexec! view-system @db
(vexec! view-system db
["update lists
set notes = ?
where id = ?"
@ -40,7 +40,7 @@
(defn update-list-visibility!
[list-id public?]
(vexec! view-system @db
(vexec! view-system db
["update lists
set is_public = ?
where id = ?"

6
src/mtgcoll/models/mtgjson.clj
View file

@ -26,7 +26,7 @@
(doseq [[_ m] json-data]
(try
(sql/insert!
@db
db
:sets
{:code (:code m)
:name (:name m)
@ -72,7 +72,7 @@
variations (:variations card)]
(try
(sql/with-db-transaction
[db-con @db]
[db-con db]
(sql/insert!
db-con
:cards
@ -127,7 +127,7 @@
[]
(println "Filling in card IDs for card variations")
(sql/execute!
@db
db
["update card_variations
set variant_card_id = (select c.id
from cards c

2
src/mtgcoll/models/sets.clj
View file

@ -5,4 +5,4 @@
(defn get-set-codes
[]
(seq (sql/query @db ["select code, gatherer_code from sets order by code"])))
(seq (sql/query db ["select code, gatherer_code from sets order by code"])))

2
src/mtgcoll/scrapers/image_assets.clj
View file

@ -30,7 +30,7 @@
(defn- get-gatherer-set-codes
[]
(sql/query @db ["select code, gatherer_code from sets order by code"]))
(sql/query db ["select code, gatherer_code from sets order by code"]))
(defn- download-set-image
[size {:keys [code gatherer_code]}]

5
src/mtgcoll/scrapers/prices.clj
View file

@ -1,18 +1,15 @@
(ns mtgcoll.scrapers.prices
(:require
[views.core :as views]
[views.sql.core :refer [hint-type]]
[mtgcoll.views.core :refer [view-system]]
[mtgcoll.models.cards :as cards]
[mtgcoll.models.sets :as sets]
[mtgcoll.scrapers.protocols :refer [scrape]]
[mtgcoll.scrapers.registered :refer [price-scrapers]]
[mtgcoll.db :as db]))
[mtgcoll.scrapers.registered :refer [price-scrapers]]))
(defn update-prices!
([source]
(println "Updating" source "card prices.")
(db/verify-connection)
(if-let [price-scraper (get price-scrapers source)]
(do
(doseq [{:keys [code gatherer_code] :as set} (sets/get-set-codes)]

31
src/mtgcoll/views/core.clj
View file

@ -1,10 +1,11 @@
(ns mtgcoll.views.core
(:require
[clojure.tools.logging :as log]
[views.reagent.sente.server :as vr]
[views.core :as views]
[views.sql.view :refer [view]]
[honeysql.format :as fmt]
[mount.core :refer [defstate]]
[views.core :as views]
[views.reagent.sente.server :as vr]
[views.sql.view :refer [view]]
[mtgcoll.db :refer [db]]
[mtgcoll.auth :as auth]
[mtgcoll.views.sente :refer [sente-socket]]
@ -15,11 +16,9 @@
[mtgcoll.views.functions.prices :as prices]
[mtgcoll.views.functions.statistics :as statistics]))
(defonce view-system (atom {}))
(defn get-db
[_]
@db)
db)
(def views
[(view :card-info get-db #'cards/card-info {:result-set-fn first})
@ -94,17 +93,15 @@
user-profile (get-in request [:session :user])]
(log/warn "Unauthorized view subscription attempt: " view-id ", " parameters " - user profile: " user-profile)))
(defn init!
[]
(vr/init! view-system @sente-socket
{:views views
:use-default-sente-router? true
:auth-fn view-auth-fn
:on-unauth-fn view-on-unauth-fn}))
(defn shutdown!
[]
(views/shutdown! view-system))
(defstate view-system
:start (let [vs (atom nil)]
(vr/init! vs sente-socket
{:views views
:use-default-sente-router? true
:auth-fn view-auth-fn
:on-unauth-fn view-on-unauth-fn})
vs)
:stop (views/shutdown! view-system))
(defmethod fmt/fn-handler "ilike"
[_ col qstr]

25
src/mtgcoll/views/sente.clj
View file

@ -1,9 +1,15 @@
(ns mtgcoll.views.sente
(:require
[mount.core :refer [defstate]]
[taoensso.sente :as sente]
[taoensso.sente.server-adapters.immutant :refer [sente-web-server-adapter]]))
(defonce sente-socket (atom {}))
(defstate sente-socket
:start (sente/make-channel-socket!
sente-web-server-adapter
{:user-id-fn (fn [request] (get-in request [:params :client-id]))
:handshake-data-fn (fn [request]
{:user (get-in request [:session :user])})}))
(defn wrap-sente
[handler uri]
@ -11,19 +17,6 @@
(let [uri-match? (.startsWith (str (:uri request)) uri)
method (:request-method request)]
(cond
(and uri-match? (= :get method)) ((:ajax-get-or-ws-handshake-fn @sente-socket) request)
(and uri-match? (= :post method)) ((:ajax-post-fn @sente-socket) request)
(and uri-match? (= :get method)) ((:ajax-get-or-ws-handshake-fn sente-socket) request)
(and uri-match? (= :post method)) ((:ajax-post-fn sente-socket) request)
:else (handler request)))))
(defn init!
[]
(reset! sente-socket
(sente/make-channel-socket!
sente-web-server-adapter
{:user-id-fn (fn [request] (get-in request [:params :client-id]))
:handshake-data-fn (fn [request]
{:user (get-in request [:session :user])})})))
(defn shutdown!
[]
(reset! sente-socket {}))