add scryfall price scraper

src/mtgcoll/models/cards.clj

@@ -28,6 +28,25 @@
                          (vec x))}]
     (seq (sql/query db (hsql/format q) {:row-fn :id}))))
 
+(defn get-card-id-by-multiverse
+  [multiverse-id set-code & [card-name split?]]
+  (seq
+    (if split?
+      (sql/query db ["select id
+                      from cards
+                      where multiverseid = ?
+                            and set_code = ?
+                            and name = ?
+                            and layout = 'split'"
+                     multiverse-id set-code card-name]
+                 {:row-fn :id})
+      (sql/query db ["select id
+                      from cards
+                      where multiverseid = ?
+                            and set_code = ?"
+                     multiverse-id set-code]
+                 {:row-fn :id}))))
+
 (defn update-price!
   [card-id price-source price online?]
   ;; written assuming postgresql server is _not_ 9.5+ (so, without access to UPSERT functionality)
@@ -55,11 +74,13 @@
         num-updates))))
 
 (defn update-prices!
-  [price-source prices & [{:keys [normalized-name?]}]]
+  [price-source prices & [{:keys [normalized-name? multiverse-id?]}]]
-  (doseq [{:keys [card-name online? set-code price] :as card-price} prices]
+  (doseq [{:keys [card-name online? set-code price multiverseid split?] :as card-price} prices]
-    (if-let [card-ids (get-matching-card-ids card-name set-code
+    (if-let [card-ids (if multiverse-id?
-                                             {:split?           (:split? card-price)
+                        (get-card-id-by-multiverse multiverseid set-code card-name split?)
-                                              :normalized-name? normalized-name?})]
+                        (get-matching-card-ids card-name set-code
+                                               {:split?           split?
+                                                :normalized-name? normalized-name?}))]
       (doseq [card-id card-ids]
         (update-price! card-id price-source price online?))
       (println "no card match found for:" card-name "," set-code))))

src/mtgcoll/scrapers/prices.clj

@@ -14,9 +14,10 @@
      (do
        (doseq [{:keys [code gatherer_code] :as set} (sets/get-set-codes)]
          (println "Scraping prices for set:" code (if gatherer_code (str "(" gatherer_code ")") ""))
-         (let [{:keys [source prices normalized-name?]} (scrape price-scraper set)]
+         (let [{:keys [source prices normalized-name? multiverse-id?]} (scrape price-scraper set)]
-           (if prices
+           (if (seq prices)
-             (cards/update-prices! source prices {:normalized-name? normalized-name?})
+             (cards/update-prices! source prices {:normalized-name? normalized-name?
+                                                  :multiverse-id?   multiverse-id?})
              (println "Could not obtain prices for set:" code (if gatherer_code (str "(" gatherer_code ")") ""))))))
      (println "No price scraper \"" source "\" found.")))
   ([]

src/mtgcoll/scrapers/prices/scryfall.clj

@@ -0,0 +1,61 @@
+(ns mtgcoll.scrapers.prices.scryfall
+  (:require
+    [clojure.string :as string]
+    [cheshire.core :as json]
+    [clj-http.client :as http]
+    [mtgcoll.scrapers.protocols :refer [PriceScraper]]
+    [mtgcoll.utils :as u]))
+
+(def ^:private http-options
+  {:headers          u/chrome-osx-request-headers
+   :throw-exceptions false})
+
+(defn- get-set-list
+  [{:keys [code]}]
+  (loop [cards []
+         page  1]
+    (let [url    (str "https://api.scryfall.com/cards/search?order=cmc&q=e:" code (if (> page 1) (str "&page=" page)))
+          result (http/get url http-options)]
+      (Thread/sleep 200)
+      (if (= 200 (:status result))
+        (let [result (json/parse-string (:body result) true)]
+          (if (:has_more result)
+            (recur (:data result) (inc page))
+            (concat cards (:data result))))))))
+
+(defrecord ScryfallPriceScraper []
+  PriceScraper
+  (scrape [_ {:keys [code] :as set}]
+    (let [cards (get-set-list set)]
+      (as-> cards x
+            (map
+              (fn [{:keys [name digital set rarity foil usd eur lang multiverse_ids collector_number]}]
+                {:card-name    name
+                 :online?      digital
+                 :set-code     code
+                 :rarity       rarity
+                 :price        (u/parse-currency-string usd)
+                 :number       collector_number
+                 :multiverseid (first multiverse_ids)})
+              x)
+            (remove
+              #(or (nil? (:price %))
+                   (nil? (:multiverseid %)))
+              x)
+            (reduce
+              (fn [coll {:keys [^String card-name price] :as card}]
+                (if (.contains card-name " // ")
+                  (let [[left right] (string/split card-name #" // ")
+                        card (assoc card :price (/ price 2)
+                                         :split? true)]
+                    (-> coll
+                        (conj (assoc card :card-name left))
+                        (conj (assoc card :card-name right))))
+                  (conj coll card)))
+              []
+              x)
+            (assoc
+              {:set-code       code
+               :source         :scryfall
+               :multiverse-id? true}
+              :prices x)))))

src/mtgcoll/scrapers/registered.clj

@@ -1,9 +1,12 @@
 (ns mtgcoll.scrapers.registered
   (:require
-    mtgcoll.scrapers.prices.mtggoldfish)
+    mtgcoll.scrapers.prices.mtggoldfish
+    mtgcoll.scrapers.prices.scryfall)
   (:import
-    (mtgcoll.scrapers.prices.mtggoldfish MTGGoldFishPriceScraper)))
+    (mtgcoll.scrapers.prices.mtggoldfish MTGGoldFishPriceScraper)
+    (mtgcoll.scrapers.prices.scryfall ScryfallPriceScraper)))
 
 (def price-scrapers
-  {:mtggoldfish (MTGGoldFishPriceScraper.)})
+  {:mtggoldfish (MTGGoldFishPriceScraper.)
+   :scryfall (ScryfallPriceScraper.)})