Add worker for downloading images

package.json

@@ -9,6 +9,7 @@
     "test:unit": "vue-cli-service test:unit"
   },
   "dependencies": {
+    "@types/jszip": "^3.1.6",
     "axios": "^0.18.0",
     "babel-core": "7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
@@ -17,6 +18,7 @@
     "core-js": "^2.6.5",
     "dexie": "^2.0.4",
     "eventemitter3": "^4.0.0",
+    "jszip": "^3.2.2",
     "node-sass": "^4.9.0",
     "peerjs": "^1.0.4",
     "register-service-worker": "^1.6.2",
@@ -28,7 +30,8 @@
     "vue-property-decorator": "^8.1.0",
     "vue-router": "^3.0.3",
     "vuex": "^3.0.1",
-    "vuex-class": "^0.3.2"
+    "vuex-class": "^0.3.2",
+    "worker-loader": "^2.0.0"
   },
   "devDependencies": {
     "@types/jest": "^23.1.4",

src/mlpccg/database.ts

@@ -19,10 +19,18 @@ class CardDatabase extends Dexie {
 
 export let Database: CardDatabase | null = null;
 
-export function initDB() {
+export async function initDB(): Promise<void> {
+  return new Promise(resolve => {
     if (Database == null) {
       Database = new CardDatabase();
+      Database.on("ready", async () => {
+        resolve();
+      });
+      Database.open();
+    } else {
+      resolve();
     }
+  });
 }
 
 export async function getCards(filter: CardFilter) {

src/mlpccg/images.ts

@@ -1,35 +1,5 @@
-import axios from "axios";
-import { Database } from "./database";
-
 const imgBaseURL = "https://mcg.zyg.ovh/images/cards/";
 
 export function cardImageURL(cardid: string): string {
   return `${imgBaseURL}${cardid}.webp`;
 }
-
-async function getCardImageList(): Promise<string[]> {
-  const req = await axios(`${imgBaseURL}list.txt`);
-  return req.data;
-}
-
-export async function getImages() {
-  if (Database == null) {
-    throw new Error("Database was not initialized, init with 'initDB()'");
-  }
-  const itemcount = await Database.images.count();
-  if (itemcount > 100) {
-    // DB already filled, exit early
-    return;
-  }
-  const imglist = await getCardImageList();
-
-  let table = Database.images;
-  const promises = imglist.map(async img => {
-    const req = await axios({
-      url: `${imgBaseURL}${img}`,
-      responseType: "blob"
-    });
-    return table.put({ id: img, image: req.data });
-  });
-  return await Promise.all(promises);
-}

src/network/Client.ts

@@ -48,7 +48,7 @@ export abstract class Client extends EventEmitter {
         this.players.push(data.name);
         this.emit("player-joined", data.name);
         break;
-      case "player-left":
+      case "player-left": {
         let idx = this.players.indexOf(data.name);
         if (idx < 0) {
           // Weird
@@ -58,8 +58,11 @@ export abstract class Client extends EventEmitter {
           break;
         }
         this.players.splice(idx, 1);
+        break;
+      }
       case "password-req":
         this.emit("password-required");
+        break;
       default:
         // For most cases, we can just use the kind as event type
         this.emit(data.kind, data);

src/tests/unit/database.spec.ts

@@ -6,7 +6,7 @@ setupIDBShim();
 describe("mlpccg/Database", () => {
   beforeAll(async () => {
     jest.setTimeout(15000);
-    initDB();
+    await initDB();
     await loadSets();
   });
 

src/tests/unit/draft.spec.ts

@@ -22,7 +22,7 @@ const testSessionOptions: DraftOptions = {
 describe("mlpccg/draft", () => {
   beforeAll(async () => {
     jest.setTimeout(15000);
-    initDB();
+    await initDB();
     await loadSets();
   });
 

src/views/Lobby.vue

@@ -16,11 +16,22 @@
 <script lang="ts">
 import { Component, Vue } from "vue-property-decorator";
 import TopNav from "@/components/Navigation/TopNav.vue";
+import DownloadWorker from "worker-loader!@/workers/tasks/downloadCardImages";
 
 @Component({
   components: {
     TopNav
   }
 })
-export default class Lobby extends Vue {}
+export default class Lobby extends Vue {
+  private mounted() {
+    /*
+    const worker = new DownloadWorker();
+    worker.addEventListener("message", (ev: MessageEvent) => {
+      const data = JSON.parse(ev.data);
+      console.log(data);
+    });
+    */
+  }
+}
 </script>

src/worker.d.ts

@@ -0,0 +1,8 @@
+// typings/custom.d.ts
+declare module "worker-loader!*" {
+  class WebpackWorker extends Worker {
+    constructor();
+  }
+
+  export default WebpackWorker;
+}

src/workers/tasks/downloadCardImages.ts

@@ -0,0 +1,95 @@
+import { Database, initDB } from "@/mlpccg";
+import axios from "axios";
+import JSZip from "jszip";
+
+const ctx: Worker = self as any;
+
+function send(data: object) {
+  ctx.postMessage(JSON.stringify(data));
+}
+
+async function downloadImages() {
+  if (!Database) {
+    await initDB();
+  }
+
+  let table = Database!.images;
+
+  const itemcount = await table.count();
+  if (itemcount > 1900) {
+    // DB already filled, exit early
+    send({
+      type: "svc-already-done"
+    });
+    return;
+  }
+
+  const zipdata = await axios({
+    url: "https://mcg.zyg.ovh/cards.zip",
+    responseType: "blob",
+    onDownloadProgress: (progressEvent: ProgressEvent) => {
+      send({
+        type: "svc-dl-progress",
+        progress: progressEvent.loaded,
+        total: progressEvent.total
+      });
+    }
+  });
+
+  const zipfile = await JSZip.loadAsync(zipdata.data);
+  const cards = zipfile.folder("Cards");
+
+  let loadingState = 0;
+  let totalLoading = 0;
+  cards.forEach(async () => {
+    totalLoading += 2;
+  });
+  let waitgroup = new Promise(resolve => {
+    let timer = setInterval(() => {
+      if (loadingState >= totalLoading) {
+        clearInterval(timer);
+        resolve();
+      }
+    }, 1000);
+  });
+  cards.forEach(async (filename, filedata) => {
+    const data = await filedata.async("blob");
+    loadingState += 1;
+    send({
+      type: "svc-ex-progress",
+      progress: loadingState,
+      total: totalLoading
+    });
+    const result = await table.put({ id: filename, image: data });
+    loadingState += 1;
+    send({
+      type: "svc-ex-progress",
+      progress: loadingState,
+      total: totalLoading
+    });
+  });
+  await waitgroup;
+}
+
+async function run() {
+  send({
+    type: "svc-start"
+  });
+
+  try {
+    await downloadImages();
+  } catch (e) {
+    console.error(e);
+    send({
+      type: "svc-err",
+      error: e.message
+    });
+    return;
+  }
+
+  send({
+    type: "svc-end"
+  });
+}
+
+run();

tsconfig.json

@@ -11,21 +11,11 @@
     "allowSyntheticDefaultImports": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "types": [
-      "webpack-env",
-      "jest"
-    ],
+    "types": ["webpack-env", "jest"],
     "paths": {
-      "@/*": [
-        "src/*"
-      ]
+      "@/*": ["src/*"]
     },
-    "lib": [
-      "esnext",
-      "dom",
-      "dom.iterable",
-      "scripthost"
-    ]
+    "lib": ["esnext", "dom", "dom.iterable", "scripthost"]
   },
   "include": [
     "src/**/*.ts",
@@ -34,7 +24,5 @@
     "tests/**/*.ts",
     "tests/**/*.tsx"
   ],
-  "exclude": [
-    "node_modules"
-  ]
+  "exclude": ["node_modules"]
 }

yarn.lock

@@ -770,6 +770,13 @@
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.3.tgz#bdfd69d61e464dcc81b25159c270d75a73c1a636"
   integrity sha512-Il2DtDVRGDcqjDtE+rF8iqg1CArehSK84HZJCT7AMITlyXRBpuPhqGLDQMowraqqu1coEaimg4ZOqggt6L6L+A==
 
+"@types/jszip@^3.1.6":
+  version "3.1.6"
+  resolved "https://registry.yarnpkg.com/@types/jszip/-/jszip-3.1.6.tgz#0512574a2b35f3194b41769c56e4b35065e67000"
+  integrity sha512-m8uFcI+O2EupCfbEVQWsBM/4nhbegjOHL7cQgBpM95FeF98kdFJXzy9/8yhx4b3lCRl/gMBhcvyh30Qt3X+XPQ==
+  dependencies:
+    "@types/node" "*"
+
 "@types/minimatch@*":
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d"
@@ -5147,6 +5154,11 @@ immediate@^3.2.2:
   resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.2.3.tgz#d140fa8f614659bd6541233097ddaac25cdd991c"
   integrity sha1-0UD6j2FGWb1lQSMwl92qwlzdmRw=
 
+immediate@~3.0.5:
+  version "3.0.6"
+  resolved "https://registry.yarnpkg.com/immediate/-/immediate-3.0.6.tgz#9db1dbd0faf8de6fbe0f5dd5e56bb606280de69b"
+  integrity sha1-nbHb0Pr43m++D13V5Wu2BigN5ps=
+
 import-cwd@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/import-cwd/-/import-cwd-2.1.0.tgz#aa6cf36e722761285cb371ec6519f53e2435b0a9"
@@ -6362,6 +6374,16 @@ jsprim@^1.2.2:
     json-schema "0.2.3"
     verror "1.10.0"
 
+jszip@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/jszip/-/jszip-3.2.2.tgz#b143816df7e106a9597a94c77493385adca5bd1d"
+  integrity sha512-NmKajvAFQpbg3taXQXr/ccS2wcucR1AZ+NtyWp2Nq7HHVsXhcJFR8p0Baf32C2yVvBylFWVeKf+WI2AnvlPhpA==
+  dependencies:
+    lie "~3.3.0"
+    pako "~1.0.2"
+    readable-stream "~2.3.6"
+    set-immediate-shim "~1.0.1"
+
 killable@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/killable/-/killable-1.0.1.tgz#4c8ce441187a061c7474fb87ca08e2a638194892"
@@ -6443,6 +6465,13 @@ levn@^0.3.0, levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+lie@~3.3.0:
+  version "3.3.0"
+  resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
+  integrity sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==
+  dependencies:
+    immediate "~3.0.5"
+
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -6482,7 +6511,7 @@ loader-utils@^0.2.16:
     json5 "^0.5.0"
     object-assign "^4.0.1"
 
-loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
+loader-utils@^1.0.0, loader-utils@^1.0.1, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
   integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==
@@ -7685,7 +7714,7 @@ p-try@^2.0.0:
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"
   integrity sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==
 
-pako@~1.0.5:
+pako@~1.0.2, pako@~1.0.5:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/pako/-/pako-1.0.10.tgz#4328badb5086a426aa90f541977d4955da5c9732"
   integrity sha512-0DTvPVU3ed8+HNXOu5Bs+o//Mbdj9VNQMUOe9oKCwh8l0GNwpTDMKCWbRjgtD291AWnkAgkqA/LOnQS8AmS1tw==
@@ -9098,6 +9127,14 @@ sax@^1.2.4, sax@~1.2.4:
   resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9"
   integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==
 
+schema-utils@^0.4.0:
+  version "0.4.7"
+  resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-0.4.7.tgz#ba74f597d2be2ea880131746ee17d0a093c68187"
+  integrity sha512-v/iwU6wvwGK8HbU9yi3/nhGzP0yGSuhQMzL6ySiec1FSrZZDkhm4noOSWzrNFo/jEc+SJY6jRTwuwbSXJPDUnQ==
+  dependencies:
+    ajv "^6.1.0"
+    ajv-keywords "^3.1.0"
+
 schema-utils@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/schema-utils/-/schema-utils-1.0.0.tgz#0b79a93204d7b600d4b2850d1f66c2a34951c770"
@@ -9204,6 +9241,11 @@ set-blocking@^2.0.0, set-blocking@~2.0.0:
   resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
   integrity sha1-BF+XgtARrppoA93TgrJDkrPYkPc=
 
+set-immediate-shim@~1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/set-immediate-shim/-/set-immediate-shim-1.0.1.tgz#4b2b1b27eb808a9f8dcc481a58e5e56f599f3f61"
+  integrity sha1-SysbJ+uAip+NzEgaWOXlb1mfP2E=
+
 set-value@^2.0.0, set-value@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/set-value/-/set-value-2.0.1.tgz#a18d40530e6f07de4228c7defe4227af8cad005b"
@@ -10979,6 +11021,14 @@ worker-farm@^1.7.0:
   dependencies:
     errno "~0.1.7"
 
+worker-loader@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/worker-loader/-/worker-loader-2.0.0.tgz#45fda3ef76aca815771a89107399ee4119b430ac"
+  integrity sha512-tnvNp4K3KQOpfRnD20m8xltE3eWh89Ye+5oj7wXEEHKac1P4oZ6p9oTj8/8ExqoSBnk9nu5Pr4nKfQ1hn2APJw==
+  dependencies:
+    loader-utils "^1.0.0"
+    schema-utils "^0.4.0"
+
 wrap-ansi@^2.0.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"