From a2d9e5bcbc8995e51acaa7738558c13822088a02 Mon Sep 17 00:00:00 2001 From: Hamcha Date: Thu, 21 Nov 2019 15:16:04 +0100 Subject: [PATCH] Add fetching/parsing some files --- .gitignore | 5 +- README.md | 8 +++ api.ts | 2 +- convert.ts | 22 +++++++ fetch.ts | 46 ++++++++++++++ main.ts | 12 ---- package.json | 5 +- update-csv.ts | 31 ---------- utils.ts | 26 ++++++++ yarn.lock | 164 +++++++++++++++++++++++++++++++++++++++++++++++++- 10 files changed, 272 insertions(+), 49 deletions(-) create mode 100644 README.md create mode 100644 convert.ts create mode 100644 fetch.ts delete mode 100644 main.ts delete mode 100644 update-csv.ts create mode 100644 utils.ts diff --git a/.gitignore b/.gitignore index a9814aa..7da4385 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ env.ps1 -node_modules \ No newline at end of file +node_modules +AllPrintings.json +mcmCards.json +*-cards.json \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..736a16b --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +# mkm junk rare finder + +## Build rare db + +To use the rare finder, you'll need to build the rare db + +1. Download this: https://www.mtgjson.com/files/AllPrintings.json +2. Run `yarn convert-db` diff --git a/api.ts b/api.ts index 61eefe2..a53bee4 100644 --- a/api.ts +++ b/api.ts @@ -16,7 +16,7 @@ export default class CardMarketApi { this.AccessSecret = process.env.ACCESS_SECRET; } - get(path: string, params?: Record): Promise { + async get(path: string, params?: Record): Promise { const oauthParameters = { realm: `${URI}${path}`, consumer_key: this.AppToken, diff --git a/convert.ts b/convert.ts new file mode 100644 index 0000000..ecdf52c --- /dev/null +++ b/convert.ts @@ -0,0 +1,22 @@ +import { asyncLoadJSON, asyncSaveJSON } from "./utils"; + +interface JSONCard { + mcmId: number; +} + +interface JSONSet { + cards: JSONCard[]; +} + +type JSONDB = Record; + +async function run() { + const db = await asyncLoadJSON("AllPrintings.json"); + let acc = {}; + for (const set in db) { + db[set].cards.forEach(c => (acc[c.mcmId] = c)); + } + asyncSaveJSON("mcmCards.json", acc); +} + +run(); diff --git a/fetch.ts b/fetch.ts new file mode 100644 index 0000000..41d5633 --- /dev/null +++ b/fetch.ts @@ -0,0 +1,46 @@ +import { existsSync } from "fs"; + +import CardMarketApi from "./api"; +import { asyncLoadJSON, asyncSaveJSON } from "./utils"; + +interface MCMCard { + name: string; + rarity: string; +} + +type MCMDB = Record; + +async function getAllArticles(api: CardMarketApi, uid: string) { + const perPage = 1000; + let start = 0; + let response; + let data = []; + do { + response = await api.get(`/users/${uid}/articles`, { + start, + maxResults: perPage + }); + data = data.concat(response.article); + start += response.article.length; + } while (response.article.length == perPage); + return data; +} + +async function run() { + if (process.argv.length < 3) { + console.error("Usage: yarn fetch "); + process.exit(1); + return; + } + const uid = process.argv[2]; + if (!existsSync("mcmCards.json")) { + console.error("Card db is missing! Run 'yarn convert-db' first."); + process.exit(1); + } + let db = await asyncLoadJSON("mcmCards.json"); + let api = new CardMarketApi(); + const articles = await getAllArticles(api, uid); + await asyncSaveJSON(`${uid}-cards.json`, articles); +} + +run(); diff --git a/main.ts b/main.ts deleted file mode 100644 index 8f64942..0000000 --- a/main.ts +++ /dev/null @@ -1,12 +0,0 @@ -import CardMarketApi from "./api"; - -async function run() { - let api = new CardMarketApi(); - const response = await api.get("/users/104821/articles", { - start: 0, - maxResults: 100 - }); - console.log(response); -} - -run(); diff --git a/package.json b/package.json index 7cd38de..4815412 100644 --- a/package.json +++ b/package.json @@ -6,11 +6,12 @@ "author": "Hamcha ", "license": "MIT", "scripts": { - "start": "ts-node main.ts", - "update-csv": "ts-node update-csv.ts" + "fetch": "ts-node fetch.ts", + "convert-db": "ts-node convert.ts" }, "dependencies": { "@types/node": "^12.12.11", + "csv-parser": "^2.3.2", "request": "^2.88.0", "ts-node": "^8.5.2", "typescript": "^3.7.2" diff --git a/update-csv.ts b/update-csv.ts deleted file mode 100644 index f9991f3..0000000 --- a/update-csv.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { createWriteStream } from "fs"; -import { gunzip } from "zlib"; - -import CardMarketApi from "./api"; - -async function run() { - let api = new CardMarketApi(); - - let data = await api.get("/productlist"); - - const gzipbuf = Buffer.from(data.productsfile, "base64"); - gunzip(gzipbuf, (err, data) => { - if (err) { - throw err; - } - const file = createWriteStream("products.csv"); - file.write(data, err => { - if (err) { - throw err; - file.close(); - } - }); - }); -} - -run(); -/* -api.req("/users/104821/articles", "start=0&maxResults=100").then(response => { - console.log(response); -}) -*/ diff --git a/utils.ts b/utils.ts new file mode 100644 index 0000000..cc333d5 --- /dev/null +++ b/utils.ts @@ -0,0 +1,26 @@ +import { readFile, writeFile } from "fs"; + +export async function asyncLoadJSON(filename: string): Promise { + return new Promise((resolve, reject) => { + readFile(filename, "utf8", (err, data) => { + if (err) { + return reject(err); + } + resolve(JSON.parse(data)); + }); + }); +} + +export async function asyncSaveJSON( + filename: string, + data: any +): Promise { + return new Promise((resolve, reject) => { + writeFile(filename, JSON.stringify(data), {}, err => { + if (err) { + return reject(err); + } + resolve(); + }); + }); +} diff --git a/yarn.lock b/yarn.lock index 86428ac..8fd2861 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,44 @@ # yarn lockfile v1 +"@hapi/address@^2.1.2": + version "2.1.4" + resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5" + integrity sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ== + +"@hapi/formula@^1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@hapi/formula/-/formula-1.2.0.tgz#994649c7fea1a90b91a0a1e6d983523f680e10cd" + integrity sha512-UFbtbGPjstz0eWHb+ga/GM3Z9EzqKXFWIbSOFURU0A/Gku0Bky4bCk9/h//K2Xr3IrCfjFNhMm4jyZ5dbCewGA== + +"@hapi/hoek@^8.2.4", "@hapi/hoek@^8.3.0": + version "8.5.0" + resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-8.5.0.tgz#2f9ce301c8898e1c3248b0a8564696b24d1a9a5a" + integrity sha512-7XYT10CZfPsH7j9F1Jmg1+d0ezOux2oM2GfArAzLwWe4mE2Dr3hVjsAL6+TFY49RRJlCdJDMw3nJsLFroTc8Kw== + +"@hapi/joi@^16.1.4": + version "16.1.7" + resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-16.1.7.tgz#360857223a87bb1f5f67691537964c1b4908ed93" + integrity sha512-anaIgnZhNooG3LJLrTFzgGALTiO97zRA1UkvQHm9KxxoSiIzCozB3RCNCpDnfhTJD72QlrHA8nwGmNgpFFCIeg== + dependencies: + "@hapi/address" "^2.1.2" + "@hapi/formula" "^1.2.0" + "@hapi/hoek" "^8.2.4" + "@hapi/pinpoint" "^1.0.2" + "@hapi/topo" "^3.1.3" + +"@hapi/pinpoint@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@hapi/pinpoint/-/pinpoint-1.0.2.tgz#025b7a36dbbf4d35bf1acd071c26b20ef41e0d13" + integrity sha512-dtXC/WkZBfC5vxscazuiJ6iq4j9oNx1SHknmIr8hofarpKUZKmlUVYVIhNVzIEgK5Wrc4GMHL5lZtt1uS2flmQ== + +"@hapi/topo@^3.1.3": + version "3.1.6" + resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-3.1.6.tgz#68d935fa3eae7fdd5ab0d7f953f3205d8b2bfc29" + integrity sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ== + dependencies: + "@hapi/hoek" "^8.3.0" + "@types/node@^12.12.11": version "12.12.11" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.11.tgz#bec2961975888d964196bf0016a2f984d793d3ce" @@ -56,6 +94,24 @@ bcrypt-pbkdf@^1.0.0: dependencies: tweetnacl "^0.14.3" +buffer-alloc-unsafe@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0" + integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg== + +buffer-alloc@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec" + integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow== + dependencies: + buffer-alloc-unsafe "^1.1.0" + buffer-fill "^1.0.0" + +buffer-fill@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c" + integrity sha1-+PeLdniYiO858gXNY39o5wISKyw= + buffer-from@^1.0.0: version "1.1.1" resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" @@ -73,11 +129,23 @@ combined-stream@^1.0.6, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" -core-util-is@1.0.2: +core-util-is@1.0.2, core-util-is@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +csv-parser@^2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-2.3.2.tgz#3a4146abb9a0be7c3cbcae9c289f0a7e622dc07b" + integrity sha512-ggurTYuhhoUJyrPXTAkiO1x6QXkxAoimV+YSz2eSAXF+jQ/Xve/030T34tVStKFmX56pPtY5PiZl3bR4HkZK+Q== + dependencies: + "@hapi/joi" "^16.1.4" + buffer-alloc "^1.1.0" + buffer-from "^1.0.0" + generate-object-property "^1.0.0" + minimist "^1.2.0" + ndjson "^1.4.0" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -142,6 +210,13 @@ form-data@~2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +generate-object-property@^1.0.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/generate-object-property/-/generate-object-property-1.2.0.tgz#9c0e1c40308ce804f4783618b937fa88f99d50d0" + integrity sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA= + dependencies: + is-property "^1.0.0" + getpass@^0.1.1: version "0.1.7" resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" @@ -171,11 +246,26 @@ http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.7.0" +inherits@~2.0.3: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +is-property@^1.0.0: + version "1.0.2" + resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84" + integrity sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ= + is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo= +isarray@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" + integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= + isstream@~0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a" @@ -196,7 +286,7 @@ json-schema@0.2.3: resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM= -json-stringify-safe@~5.0.1: +json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb" integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus= @@ -228,6 +318,21 @@ mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.42.0" +minimist@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" + integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ= + +ndjson@^1.4.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8" + integrity sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg= + dependencies: + json-stringify-safe "^5.0.1" + minimist "^1.2.0" + split2 "^2.1.0" + through2 "^2.0.3" + oauth-sign@~0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" @@ -238,6 +343,11 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= +process-nextick-args@~2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2" + integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag== + psl@^1.1.24: version "1.4.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.4.0.tgz#5dd26156cdb69fa1fdb8ab1991667d3f80ced7c2" @@ -258,6 +368,19 @@ qs@~6.5.2: resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA== +readable-stream@~2.3.6: + version "2.3.6" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" + integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw== + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.3" + isarray "~1.0.0" + process-nextick-args "~2.0.0" + safe-buffer "~5.1.1" + string_decoder "~1.1.1" + util-deprecate "~1.0.1" + request@^2.88.0: version "2.88.0" resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" @@ -289,6 +412,11 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.2: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519" integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg== +safe-buffer@~5.1.0, safe-buffer@~5.1.1: + version "5.1.2" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== + safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" @@ -307,6 +435,13 @@ source-map@^0.6.0: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +split2@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493" + integrity sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw== + dependencies: + through2 "^2.0.2" + sshpk@^1.7.0: version "1.16.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" @@ -322,6 +457,21 @@ sshpk@^1.7.0: safer-buffer "^2.0.2" tweetnacl "~0.14.0" +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +through2@^2.0.2, through2@^2.0.3: + version "2.0.5" + resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd" + integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ== + dependencies: + readable-stream "~2.3.6" + xtend "~4.0.1" + tough-cookie@~2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" @@ -365,6 +515,11 @@ uri-js@^4.2.2: dependencies: punycode "^2.1.0" +util-deprecate@~1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" + integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + uuid@^3.3.2: version "3.3.3" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866" @@ -379,6 +534,11 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" +xtend@~4.0.1: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== + yn@^3.0.0: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"