From 5b8cb759216b7c6ff8829bb351bee3778e0b25bb Mon Sep 17 00:00:00 2001 From: Hamcha Date: Wed, 18 Sep 2019 13:04:43 +0200 Subject: [PATCH] Add worker for downloading images --- package.json | 5 +- src/mlpccg/database.ts | 16 +++-- src/mlpccg/images.ts | 30 -------- src/network/Client.ts | 5 +- src/tests/unit/database.spec.ts | 2 +- src/tests/unit/draft.spec.ts | 2 +- src/views/Lobby.vue | 13 +++- src/worker.d.ts | 8 +++ src/workers/tasks/downloadCardImages.ts | 95 +++++++++++++++++++++++++ tsconfig.json | 22 ++---- yarn.lock | 54 +++++++++++++- 11 files changed, 194 insertions(+), 58 deletions(-) create mode 100644 src/worker.d.ts create mode 100644 src/workers/tasks/downloadCardImages.ts diff --git a/package.json b/package.json index 953faba..9ed7263 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "test:unit": "vue-cli-service test:unit" }, "dependencies": { + "@types/jszip": "^3.1.6", "axios": "^0.18.0", "babel-core": "7.0.0-bridge.0", "babel-eslint": "^10.0.1", @@ -17,6 +18,7 @@ "core-js": "^2.6.5", "dexie": "^2.0.4", "eventemitter3": "^4.0.0", + "jszip": "^3.2.2", "node-sass": "^4.9.0", "peerjs": "^1.0.4", "register-service-worker": "^1.6.2", @@ -28,7 +30,8 @@ "vue-property-decorator": "^8.1.0", "vue-router": "^3.0.3", "vuex": "^3.0.1", - "vuex-class": "^0.3.2" + "vuex-class": "^0.3.2", + "worker-loader": "^2.0.0" }, "devDependencies": { "@types/jest": "^23.1.4", diff --git a/src/mlpccg/database.ts b/src/mlpccg/database.ts index c4b9aae..b2d03e2 100644 --- a/src/mlpccg/database.ts +++ b/src/mlpccg/database.ts @@ -19,10 +19,18 @@ class CardDatabase extends Dexie { export let Database: CardDatabase | null = null; -export function initDB() { - if (Database == null) { - Database = new CardDatabase(); - } +export async function initDB(): Promise { + return new Promise(resolve => { + if (Database == null) { + Database = new CardDatabase(); + Database.on("ready", async () => { + resolve(); + }); + Database.open(); + } else { + resolve(); + } + }); } export async function getCards(filter: CardFilter) { diff --git a/src/mlpccg/images.ts b/src/mlpccg/images.ts index eb35fb1..00966da 100644 --- a/src/mlpccg/images.ts +++ b/src/mlpccg/images.ts @@ -1,35 +1,5 @@ -import axios from "axios"; -import { Database } from "./database"; - const imgBaseURL = "https://mcg.zyg.ovh/images/cards/"; export function cardImageURL(cardid: string): string { return `${imgBaseURL}${cardid}.webp`; } - -async function getCardImageList(): Promise { - const req = await axios(`${imgBaseURL}list.txt`); - return req.data; -} - -export async function getImages() { - if (Database == null) { - throw new Error("Database was not initialized, init with 'initDB()'"); - } - const itemcount = await Database.images.count(); - if (itemcount > 100) { - // DB already filled, exit early - return; - } - const imglist = await getCardImageList(); - - let table = Database.images; - const promises = imglist.map(async img => { - const req = await axios({ - url: `${imgBaseURL}${img}`, - responseType: "blob" - }); - return table.put({ id: img, image: req.data }); - }); - return await Promise.all(promises); -} diff --git a/src/network/Client.ts b/src/network/Client.ts index a6a43c1..d8da32d 100644 --- a/src/network/Client.ts +++ b/src/network/Client.ts @@ -48,7 +48,7 @@ export abstract class Client extends EventEmitter { this.players.push(data.name); this.emit("player-joined", data.name); break; - case "player-left": + case "player-left": { let idx = this.players.indexOf(data.name); if (idx < 0) { // Weird @@ -58,8 +58,11 @@ export abstract class Client extends EventEmitter { break; } this.players.splice(idx, 1); + break; + } case "password-req": this.emit("password-required"); + break; default: // For most cases, we can just use the kind as event type this.emit(data.kind, data); diff --git a/src/tests/unit/database.spec.ts b/src/tests/unit/database.spec.ts index d1d77ff..d1ecec4 100644 --- a/src/tests/unit/database.spec.ts +++ b/src/tests/unit/database.spec.ts @@ -6,7 +6,7 @@ setupIDBShim(); describe("mlpccg/Database", () => { beforeAll(async () => { jest.setTimeout(15000); - initDB(); + await initDB(); await loadSets(); }); diff --git a/src/tests/unit/draft.spec.ts b/src/tests/unit/draft.spec.ts index 1db04e2..d96a3d4 100644 --- a/src/tests/unit/draft.spec.ts +++ b/src/tests/unit/draft.spec.ts @@ -22,7 +22,7 @@ const testSessionOptions: DraftOptions = { describe("mlpccg/draft", () => { beforeAll(async () => { jest.setTimeout(15000); - initDB(); + await initDB(); await loadSets(); }); diff --git a/src/views/Lobby.vue b/src/views/Lobby.vue index 041677a..51779a7 100644 --- a/src/views/Lobby.vue +++ b/src/views/Lobby.vue @@ -16,11 +16,22 @@ diff --git a/src/worker.d.ts b/src/worker.d.ts new file mode 100644 index 0000000..dcebcff --- /dev/null +++ b/src/worker.d.ts @@ -0,0 +1,8 @@ +// typings/custom.d.ts +declare module "worker-loader!*" { + class WebpackWorker extends Worker { + constructor(); + } + + export default WebpackWorker; +} diff --git a/src/workers/tasks/downloadCardImages.ts b/src/workers/tasks/downloadCardImages.ts new file mode 100644 index 0000000..e93778c --- /dev/null +++ b/src/workers/tasks/downloadCardImages.ts @@ -0,0 +1,95 @@ +import { Database, initDB } from "@/mlpccg"; +import axios from "axios"; +import JSZip from "jszip"; + +const ctx: Worker = self as any; + +function send(data: object) { + ctx.postMessage(JSON.stringify(data)); +} + +async function downloadImages() { + if (!Database) { + await initDB(); + } + + let table = Database!.images; + + const itemcount = await table.count(); + if (itemcount > 1900) { + // DB already filled, exit early + send({ + type: "svc-already-done" + }); + return; + } + + const zipdata = await axios({ + url: "https://mcg.zyg.ovh/cards.zip", + responseType: "blob", + onDownloadProgress: (progressEvent: ProgressEvent) => { + send({ + type: "svc-dl-progress", + progress: progressEvent.loaded, + total: progressEvent.total + }); + } + }); + + const zipfile = await JSZip.loadAsync(zipdata.data); + const cards = zipfile.folder("Cards"); + + let loadingState = 0; + let totalLoading = 0; + cards.forEach(async () => { + totalLoading += 2; + }); + let waitgroup = new Promise(resolve => { + let timer = setInterval(() => { + if (loadingState >= totalLoading) { + clearInterval(timer); + resolve(); + } + }, 1000); + }); + cards.forEach(async (filename, filedata) => { + const data = await filedata.async("blob"); + loadingState += 1; + send({ + type: "svc-ex-progress", + progress: loadingState, + total: totalLoading + }); + const result = await table.put({ id: filename, image: data }); + loadingState += 1; + send({ + type: "svc-ex-progress", + progress: loadingState, + total: totalLoading + }); + }); + await waitgroup; +} + +async function run() { + send({ + type: "svc-start" + }); + + try { + await downloadImages(); + } catch (e) { + console.error(e); + send({ + type: "svc-err", + error: e.message + }); + return; + } + + send({ + type: "svc-end" + }); +} + +run(); diff --git a/tsconfig.json b/tsconfig.json index 52cfe29..a2eba52 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,21 +11,11 @@ "allowSyntheticDefaultImports": true, "sourceMap": true, "baseUrl": ".", - "types": [ - "webpack-env", - "jest" - ], + "types": ["webpack-env", "jest"], "paths": { - "@/*": [ - "src/*" - ] + "@/*": ["src/*"] }, - "lib": [ - "esnext", - "dom", - "dom.iterable", - "scripthost" - ] + "lib": ["esnext", "dom", "dom.iterable", "scripthost"] }, "include": [ "src/**/*.ts", @@ -34,7 +24,5 @@ "tests/**/*.ts", "tests/**/*.tsx" ], - "exclude": [ - "node_modules" - ] -} \ No newline at end of file + "exclude": ["node_modules"] +} diff --git a/yarn.lock b/yarn.lock index fdfc99e..e22f988 100644 --- a/yarn.lock +++ b/yarn.lock @@ -770,6 +770,13 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636" integrity sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A== +"@types/jszip@^3.1.6": + version "3.1.6" + resolved "https://registry.yarnpkg.com/@types/jszip/-/jszip-3.1.6.tgz#0512574a2b35f3194b41769c56e4b35065e67000" + integrity sha512-m8uFcI+O2EupCfbEVQWsBM/4nhbegjOHL7cQgBpM95FeF98kdFJXzy9/8yhx4b3lCRl/gMBhcvyh30Qt3X+XPQ== + dependencies: + "@types/node" "*" + "@types/minimatch@*": version "3.0.3" resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" @@ -5147,6 +5154,11 @@ immediate@^3.2.2: resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.2.3.tgz#d140fa8f614659bd6541233097ddaac25cdd991c" integrity sha1-0UD6j2FGWb1lQSMwl92qwlzdmRw= +immediate@~3.0.5: + version "3.0.6" + resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b" + integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps= + import-cwd@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/import-cwd/-/import-cwd-2.1.0.tgz#aa6cf36e722761285cb371ec6519f53e2435b0a9" @@ -6362,6 +6374,16 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" +jszip@^3.2.2: + version "3.2.2" + resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.2.2.tgz#b143816df7e106a9597a94c77493385adca5bd1d" + integrity sha512-NmKajvAFQpbg3taXQXr/ccS2wcucR1AZ+NtyWp2Nq7HHVsXhcJFR8p0Baf32C2yVvBylFWVeKf+WI2AnvlPhpA== + dependencies: + lie "~3.3.0" + pako "~1.0.2" + readable-stream "~2.3.6" + set-immediate-shim "~1.0.1" + killable@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/killable/-/killable-1.0.1.tgz#4c8ce441187a061c7474fb87ca08e2a638194892" @@ -6443,6 +6465,13 @@ levn@^0.3.0, levn@~0.3.0: prelude-ls "~1.1.2" type-check "~0.3.2" +lie@~3.3.0: + version "3.3.0" + resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a" + integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ== + dependencies: + immediate "~3.0.5" + lines-and-columns@^1.1.6: version "1.1.6" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00" @@ -6482,7 +6511,7 @@ loader-utils@^0.2.16: json5 "^0.5.0" object-assign "^4.0.1" -loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3: +loader-utils@^1.0.0, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7" integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA== @@ -7685,7 +7714,7 @@ p-try@^2.0.0: resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6" integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ== -pako@~1.0.5: +pako@~1.0.2, pako@~1.0.5: version "1.0.10" resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.10.tgz#4328badb5086a426aa90f541977d4955da5c9732" integrity sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw== @@ -9098,6 +9127,14 @@ sax@^1.2.4, sax@~1.2.4: resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== +schema-utils@^0.4.0: + version "0.4.7" + resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187" + integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ== + dependencies: + ajv "^6.1.0" + ajv-keywords "^3.1.0" + schema-utils@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770" @@ -9204,6 +9241,11 @@ set-blocking@^2.0.0, set-blocking@~2.0.0: resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc= +set-immediate-shim@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61" + integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E= + set-value@^2.0.0, set-value@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b" @@ -10979,6 +11021,14 @@ worker-farm@^1.7.0: dependencies: errno "~0.1.7" +worker-loader@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-2.0.0.tgz#45fda3ef76aca815771a89107399ee4119b430ac" + integrity sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw== + dependencies: + loader-utils "^1.0.0" + schema-utils "^0.4.0" + wrap-ansi@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"