From 3c511f710b8aad62bf872f14366f1bd6271c0b39 Mon Sep 17 00:00:00 2001 From: gered Date: Wed, 13 Jun 2018 19:44:18 -0400 Subject: [PATCH] add scryfall price scraper --- src/mtgcoll/models/cards.clj | 31 ++++++++++-- src/mtgcoll/scrapers/prices.clj | 7 +-- src/mtgcoll/scrapers/prices/scryfall.clj | 61 ++++++++++++++++++++++++ src/mtgcoll/scrapers/registered.clj | 9 ++-- 4 files changed, 97 insertions(+), 11 deletions(-) create mode 100644 src/mtgcoll/scrapers/prices/scryfall.clj diff --git a/src/mtgcoll/models/cards.clj b/src/mtgcoll/models/cards.clj index f70db6b..ac98653 100644 --- a/src/mtgcoll/models/cards.clj +++ b/src/mtgcoll/models/cards.clj @@ -28,6 +28,25 @@ (vec x))}] (seq (sql/query db (hsql/format q) {:row-fn :id})))) +(defn get-card-id-by-multiverse + [multiverse-id set-code & [card-name split?]] + (seq + (if split? + (sql/query db ["select id + from cards + where multiverseid = ? + and set_code = ? + and name = ? + and layout = 'split'" + multiverse-id set-code card-name] + {:row-fn :id}) + (sql/query db ["select id + from cards + where multiverseid = ? + and set_code = ?" + multiverse-id set-code] + {:row-fn :id})))) + (defn update-price! [card-id price-source price online?] ;; written assuming postgresql server is _not_ 9.5+ (so, without access to UPSERT functionality) @@ -55,11 +74,13 @@ num-updates)))) (defn update-prices! - [price-source prices & [{:keys [normalized-name?]}]] - (doseq [{:keys [card-name online? set-code price] :as card-price} prices] - (if-let [card-ids (get-matching-card-ids card-name set-code - {:split? (:split? card-price) - :normalized-name? normalized-name?})] + [price-source prices & [{:keys [normalized-name? multiverse-id?]}]] + (doseq [{:keys [card-name online? set-code price multiverseid split?] :as card-price} prices] + (if-let [card-ids (if multiverse-id? + (get-card-id-by-multiverse multiverseid set-code card-name split?) + (get-matching-card-ids card-name set-code + {:split? split? + :normalized-name? normalized-name?}))] (doseq [card-id card-ids] (update-price! card-id price-source price online?)) (println "no card match found for:" card-name "," set-code)))) diff --git a/src/mtgcoll/scrapers/prices.clj b/src/mtgcoll/scrapers/prices.clj index 16eb256..ea1a71f 100644 --- a/src/mtgcoll/scrapers/prices.clj +++ b/src/mtgcoll/scrapers/prices.clj @@ -14,9 +14,10 @@ (do (doseq [{:keys [code gatherer_code] :as set} (sets/get-set-codes)] (println "Scraping prices for set:" code (if gatherer_code (str "(" gatherer_code ")") "")) - (let [{:keys [source prices normalized-name?]} (scrape price-scraper set)] - (if prices - (cards/update-prices! source prices {:normalized-name? normalized-name?}) + (let [{:keys [source prices normalized-name? multiverse-id?]} (scrape price-scraper set)] + (if (seq prices) + (cards/update-prices! source prices {:normalized-name? normalized-name? + :multiverse-id? multiverse-id?}) (println "Could not obtain prices for set:" code (if gatherer_code (str "(" gatherer_code ")") "")))))) (println "No price scraper \"" source "\" found."))) ([] diff --git a/src/mtgcoll/scrapers/prices/scryfall.clj b/src/mtgcoll/scrapers/prices/scryfall.clj new file mode 100644 index 0000000..e3904fd --- /dev/null +++ b/src/mtgcoll/scrapers/prices/scryfall.clj @@ -0,0 +1,61 @@ +(ns mtgcoll.scrapers.prices.scryfall + (:require + [clojure.string :as string] + [cheshire.core :as json] + [clj-http.client :as http] + [mtgcoll.scrapers.protocols :refer [PriceScraper]] + [mtgcoll.utils :as u])) + +(def ^:private http-options + {:headers u/chrome-osx-request-headers + :throw-exceptions false}) + +(defn- get-set-list + [{:keys [code]}] + (loop [cards [] + page 1] + (let [url (str "https://api.scryfall.com/cards/search?order=cmc&q=e:" code (if (> page 1) (str "&page=" page))) + result (http/get url http-options)] + (Thread/sleep 200) + (if (= 200 (:status result)) + (let [result (json/parse-string (:body result) true)] + (if (:has_more result) + (recur (:data result) (inc page)) + (concat cards (:data result)))))))) + +(defrecord ScryfallPriceScraper [] + PriceScraper + (scrape [_ {:keys [code] :as set}] + (let [cards (get-set-list set)] + (as-> cards x + (map + (fn [{:keys [name digital set rarity foil usd eur lang multiverse_ids collector_number]}] + {:card-name name + :online? digital + :set-code code + :rarity rarity + :price (u/parse-currency-string usd) + :number collector_number + :multiverseid (first multiverse_ids)}) + x) + (remove + #(or (nil? (:price %)) + (nil? (:multiverseid %))) + x) + (reduce + (fn [coll {:keys [^String card-name price] :as card}] + (if (.contains card-name " // ") + (let [[left right] (string/split card-name #" // ") + card (assoc card :price (/ price 2) + :split? true)] + (-> coll + (conj (assoc card :card-name left)) + (conj (assoc card :card-name right)))) + (conj coll card))) + [] + x) + (assoc + {:set-code code + :source :scryfall + :multiverse-id? true} + :prices x))))) diff --git a/src/mtgcoll/scrapers/registered.clj b/src/mtgcoll/scrapers/registered.clj index 66bb73c..7ea5e14 100644 --- a/src/mtgcoll/scrapers/registered.clj +++ b/src/mtgcoll/scrapers/registered.clj @@ -1,9 +1,12 @@ (ns mtgcoll.scrapers.registered (:require - mtgcoll.scrapers.prices.mtggoldfish) + mtgcoll.scrapers.prices.mtggoldfish + mtgcoll.scrapers.prices.scryfall) (:import - (mtgcoll.scrapers.prices.mtggoldfish MTGGoldFishPriceScraper))) + (mtgcoll.scrapers.prices.mtggoldfish MTGGoldFishPriceScraper) + (mtgcoll.scrapers.prices.scryfall ScryfallPriceScraper))) (def price-scrapers - {:mtggoldfish (MTGGoldFishPriceScraper.)}) + {:mtggoldfish (MTGGoldFishPriceScraper.) + :scryfall (ScryfallPriceScraper.)})