Add fetching/parsing some files

This commit is contained in:
Hamcha 2019-11-21 15:16:04 +01:00
parent 5d0c3abd5f
commit a2d9e5bcbc
Signed by: hamcha
GPG key ID: 44AD3571EB09A39E
10 changed files with 272 additions and 49 deletions

5
.gitignore vendored
View file

@ -1,2 +1,5 @@
env.ps1
node_modules
node_modules
AllPrintings.json
mcmCards.json
*-cards.json

8
README.md Normal file
View file

@ -0,0 +1,8 @@
# mkm junk rare finder
## Build rare db
To use the rare finder, you'll need to build the rare db
1. Download this: https://www.mtgjson.com/files/AllPrintings.json
2. Run `yarn convert-db`

2
api.ts
View file

@ -16,7 +16,7 @@ export default class CardMarketApi {
this.AccessSecret = process.env.ACCESS_SECRET;
}
get(path: string, params?: Record<string, any>): Promise<any> {
async get(path: string, params?: Record<string, any>): Promise<any> {
const oauthParameters = {
realm: `${URI}${path}`,
consumer_key: this.AppToken,

22
convert.ts Normal file
View file

@ -0,0 +1,22 @@
import { asyncLoadJSON, asyncSaveJSON } from "./utils";
interface JSONCard {
mcmId: number;
}
interface JSONSet {
cards: JSONCard[];
}
type JSONDB = Record<string, JSONSet>;
async function run() {
const db = await asyncLoadJSON<JSONDB>("AllPrintings.json");
let acc = {};
for (const set in db) {
db[set].cards.forEach(c => (acc[c.mcmId] = c));
}
asyncSaveJSON("mcmCards.json", acc);
}
run();

46
fetch.ts Normal file
View file

@ -0,0 +1,46 @@
import { existsSync } from "fs";
import CardMarketApi from "./api";
import { asyncLoadJSON, asyncSaveJSON } from "./utils";
interface MCMCard {
name: string;
rarity: string;
}
type MCMDB = Record<string, MCMCard>;
async function getAllArticles(api: CardMarketApi, uid: string) {
const perPage = 1000;
let start = 0;
let response;
let data = [];
do {
response = await api.get(`/users/${uid}/articles`, {
start,
maxResults: perPage
});
data = data.concat(response.article);
start += response.article.length;
} while (response.article.length == perPage);
return data;
}
async function run() {
if (process.argv.length < 3) {
console.error("Usage: yarn fetch <uid>");
process.exit(1);
return;
}
const uid = process.argv[2];
if (!existsSync("mcmCards.json")) {
console.error("Card db is missing! Run 'yarn convert-db' first.");
process.exit(1);
}
let db = await asyncLoadJSON<MCMDB>("mcmCards.json");
let api = new CardMarketApi();
const articles = await getAllArticles(api, uid);
await asyncSaveJSON(`${uid}-cards.json`, articles);
}
run();

12
main.ts
View file

@ -1,12 +0,0 @@
import CardMarketApi from "./api";
async function run() {
let api = new CardMarketApi();
const response = await api.get("/users/104821/articles", {
start: 0,
maxResults: 100
});
console.log(response);
}
run();

View file

@ -6,11 +6,12 @@
"author": "Hamcha <hamcha@crunchy.rocks>",
"license": "MIT",
"scripts": {
"start": "ts-node main.ts",
"update-csv": "ts-node update-csv.ts"
"fetch": "ts-node fetch.ts",
"convert-db": "ts-node convert.ts"
},
"dependencies": {
"@types/node": "^12.12.11",
"csv-parser": "^2.3.2",
"request": "^2.88.0",
"ts-node": "^8.5.2",
"typescript": "^3.7.2"

View file

@ -1,31 +0,0 @@
import { createWriteStream } from "fs";
import { gunzip } from "zlib";
import CardMarketApi from "./api";
async function run() {
let api = new CardMarketApi();
let data = await api.get("/productlist");
const gzipbuf = Buffer.from(data.productsfile, "base64");
gunzip(gzipbuf, (err, data) => {
if (err) {
throw err;
}
const file = createWriteStream("products.csv");
file.write(data, err => {
if (err) {
throw err;
file.close();
}
});
});
}
run();
/*
api.req("/users/104821/articles", "start=0&maxResults=100").then(response => {
console.log(response);
})
*/

26
utils.ts Normal file
View file

@ -0,0 +1,26 @@
import { readFile, writeFile } from "fs";
export async function asyncLoadJSON<T>(filename: string): Promise<T> {
return new Promise((resolve, reject) => {
readFile(filename, "utf8", (err, data) => {
if (err) {
return reject(err);
}
resolve(JSON.parse(data));
});
});
}
export async function asyncSaveJSON(
filename: string,
data: any
): Promise<void> {
return new Promise((resolve, reject) => {
writeFile(filename, JSON.stringify(data), {}, err => {
if (err) {
return reject(err);
}
resolve();
});
});
}

164
yarn.lock
View file

@ -2,6 +2,44 @@
# yarn lockfile v1
"@hapi/address@^2.1.2":
version "2.1.4"
resolved "https://registry.yarnpkg.com/@hapi/address/-/address-2.1.4.tgz#5d67ed43f3fd41a69d4b9ff7b56e7c0d1d0a81e5"
integrity sha512-QD1PhQk+s31P1ixsX0H0Suoupp3VMXzIVMSwobR3F3MSUO2YCV0B7xqLcUw/Bh8yuvd3LhpyqLQWTNcRmp6IdQ==
"@hapi/formula@^1.2.0":
version "1.2.0"
resolved "https://registry.yarnpkg.com/@hapi/formula/-/formula-1.2.0.tgz#994649c7fea1a90b91a0a1e6d983523f680e10cd"
integrity sha512-UFbtbGPjstz0eWHb+ga/GM3Z9EzqKXFWIbSOFURU0A/Gku0Bky4bCk9/h//K2Xr3IrCfjFNhMm4jyZ5dbCewGA==
"@hapi/hoek@^8.2.4", "@hapi/hoek@^8.3.0":
version "8.5.0"
resolved "https://registry.yarnpkg.com/@hapi/hoek/-/hoek-8.5.0.tgz#2f9ce301c8898e1c3248b0a8564696b24d1a9a5a"
integrity sha512-7XYT10CZfPsH7j9F1Jmg1+d0ezOux2oM2GfArAzLwWe4mE2Dr3hVjsAL6+TFY49RRJlCdJDMw3nJsLFroTc8Kw==
"@hapi/joi@^16.1.4":
version "16.1.7"
resolved "https://registry.yarnpkg.com/@hapi/joi/-/joi-16.1.7.tgz#360857223a87bb1f5f67691537964c1b4908ed93"
integrity sha512-anaIgnZhNooG3LJLrTFzgGALTiO97zRA1UkvQHm9KxxoSiIzCozB3RCNCpDnfhTJD72QlrHA8nwGmNgpFFCIeg==
dependencies:
"@hapi/address" "^2.1.2"
"@hapi/formula" "^1.2.0"
"@hapi/hoek" "^8.2.4"
"@hapi/pinpoint" "^1.0.2"
"@hapi/topo" "^3.1.3"
"@hapi/pinpoint@^1.0.2":
version "1.0.2"
resolved "https://registry.yarnpkg.com/@hapi/pinpoint/-/pinpoint-1.0.2.tgz#025b7a36dbbf4d35bf1acd071c26b20ef41e0d13"
integrity sha512-dtXC/WkZBfC5vxscazuiJ6iq4j9oNx1SHknmIr8hofarpKUZKmlUVYVIhNVzIEgK5Wrc4GMHL5lZtt1uS2flmQ==
"@hapi/topo@^3.1.3":
version "3.1.6"
resolved "https://registry.yarnpkg.com/@hapi/topo/-/topo-3.1.6.tgz#68d935fa3eae7fdd5ab0d7f953f3205d8b2bfc29"
integrity sha512-tAag0jEcjwH+P2quUfipd7liWCNX2F8NvYjQp2wtInsZxnMlypdw0FtAOLxtvvkO+GSRRbmNi8m/5y42PQJYCQ==
dependencies:
"@hapi/hoek" "^8.3.0"
"@types/node@^12.12.11":
version "12.12.11"
resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.11.tgz#bec2961975888d964196bf0016a2f984d793d3ce"
@ -56,6 +94,24 @@ bcrypt-pbkdf@^1.0.0:
dependencies:
tweetnacl "^0.14.3"
buffer-alloc-unsafe@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz#bd7dc26ae2972d0eda253be061dba992349c19f0"
integrity sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==
buffer-alloc@^1.1.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/buffer-alloc/-/buffer-alloc-1.2.0.tgz#890dd90d923a873e08e10e5fd51a57e5b7cce0ec"
integrity sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==
dependencies:
buffer-alloc-unsafe "^1.1.0"
buffer-fill "^1.0.0"
buffer-fill@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/buffer-fill/-/buffer-fill-1.0.0.tgz#f8f78b76789888ef39f205cd637f68e702122b2c"
integrity sha1-+PeLdniYiO858gXNY39o5wISKyw=
buffer-from@^1.0.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef"
@ -73,11 +129,23 @@ combined-stream@^1.0.6, combined-stream@~1.0.6:
dependencies:
delayed-stream "~1.0.0"
core-util-is@1.0.2:
core-util-is@1.0.2, core-util-is@~1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=
csv-parser@^2.3.2:
version "2.3.2"
resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-2.3.2.tgz#3a4146abb9a0be7c3cbcae9c289f0a7e622dc07b"
integrity sha512-ggurTYuhhoUJyrPXTAkiO1x6QXkxAoimV+YSz2eSAXF+jQ/Xve/030T34tVStKFmX56pPtY5PiZl3bR4HkZK+Q==
dependencies:
"@hapi/joi" "^16.1.4"
buffer-alloc "^1.1.0"
buffer-from "^1.0.0"
generate-object-property "^1.0.0"
minimist "^1.2.0"
ndjson "^1.4.0"
dashdash@^1.12.0:
version "1.14.1"
resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0"
@ -142,6 +210,13 @@ form-data@~2.3.2:
combined-stream "^1.0.6"
mime-types "^2.1.12"
generate-object-property@^1.0.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/generate-object-property/-/generate-object-property-1.2.0.tgz#9c0e1c40308ce804f4783618b937fa88f99d50d0"
integrity sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=
dependencies:
is-property "^1.0.0"
getpass@^0.1.1:
version "0.1.7"
resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa"
@ -171,11 +246,26 @@ http-signature@~1.2.0:
jsprim "^1.2.2"
sshpk "^1.7.0"
inherits@~2.0.3:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
is-property@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84"
integrity sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=
is-typedarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=
isstream@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
@ -196,7 +286,7 @@ json-schema@0.2.3:
resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
integrity sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=
json-stringify-safe@~5.0.1:
json-stringify-safe@^5.0.1, json-stringify-safe@~5.0.1:
version "5.0.1"
resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
integrity sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=
@ -228,6 +318,21 @@ mime-types@^2.1.12, mime-types@~2.1.19:
dependencies:
mime-db "1.42.0"
minimist@^1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
integrity sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=
ndjson@^1.4.0:
version "1.5.0"
resolved "https://registry.yarnpkg.com/ndjson/-/ndjson-1.5.0.tgz#ae603b36b134bcec347b452422b0bf98d5832ec8"
integrity sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=
dependencies:
json-stringify-safe "^5.0.1"
minimist "^1.2.0"
split2 "^2.1.0"
through2 "^2.0.3"
oauth-sign@~0.9.0:
version "0.9.0"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455"
@ -238,6 +343,11 @@ performance-now@^2.1.0:
resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=
process-nextick-args@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
psl@^1.1.24:
version "1.4.0"
resolved "https://registry.yarnpkg.com/psl/-/psl-1.4.0.tgz#5dd26156cdb69fa1fdb8ab1991667d3f80ced7c2"
@ -258,6 +368,19 @@ qs@~6.5.2:
resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36"
integrity sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==
readable-stream@~2.3.6:
version "2.3.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
integrity sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==
dependencies:
core-util-is "~1.0.0"
inherits "~2.0.3"
isarray "~1.0.0"
process-nextick-args "~2.0.0"
safe-buffer "~5.1.1"
string_decoder "~1.1.1"
util-deprecate "~1.0.1"
request@^2.88.0:
version "2.88.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef"
@ -289,6 +412,11 @@ safe-buffer@^5.0.1, safe-buffer@^5.1.2:
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.0.tgz#b74daec49b1148f88c64b68d49b1e815c1f2f519"
integrity sha512-fZEwUGbVl7kouZs1jCdMLdt95hdIv0ZeHg6L7qPeciMZhZ+/gdesW4wgTARkrFWEpspjEATAzUGPG8N2jJiwbg==
safe-buffer@~5.1.0, safe-buffer@~5.1.1:
version "5.1.2"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==
safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
version "2.1.2"
resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@ -307,6 +435,13 @@ source-map@^0.6.0:
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263"
integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==
split2@^2.1.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/split2/-/split2-2.2.0.tgz#186b2575bcf83e85b7d18465756238ee4ee42493"
integrity sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==
dependencies:
through2 "^2.0.2"
sshpk@^1.7.0:
version "1.16.1"
resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877"
@ -322,6 +457,21 @@ sshpk@^1.7.0:
safer-buffer "^2.0.2"
tweetnacl "~0.14.0"
string_decoder@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==
dependencies:
safe-buffer "~5.1.0"
through2@^2.0.2, through2@^2.0.3:
version "2.0.5"
resolved "https://registry.yarnpkg.com/through2/-/through2-2.0.5.tgz#01c1e39eb31d07cb7d03a96a70823260b23132cd"
integrity sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==
dependencies:
readable-stream "~2.3.6"
xtend "~4.0.1"
tough-cookie@~2.4.3:
version "2.4.3"
resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781"
@ -365,6 +515,11 @@ uri-js@^4.2.2:
dependencies:
punycode "^2.1.0"
util-deprecate@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
uuid@^3.3.2:
version "3.3.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.3.tgz#4568f0216e78760ee1dbf3a4d2cf53e224112866"
@ -379,6 +534,11 @@ verror@1.10.0:
core-util-is "1.0.2"
extsprintf "^1.2.0"
xtend@~4.0.1:
version "4.0.2"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
yn@^3.0.0:
version "3.1.1"
resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50"