moved general file opening logic to a central location

decode.py

@@ -43,79 +43,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std',
     else:
         raise ValueError('encode.py: unknown encoding: ' + encoding)
 
-    cards = []
+    cards = jdecode.mtg_open_file(fname, verbose=verbose, fmt_ordered=fmt_ordered)
-    valid = 0
-    invalid = 0
-    unparsed = 0
-
-    if fname[-5:] == '.json':
-        if verbose:
-            print 'This looks like a json file: ' + fname
-        json_srcs = jdecode.mtg_open_json(fname, verbose)
-        for json_cardname in sorted(json_srcs):
-            if len(json_srcs[json_cardname]) > 0:
-                jcards = json_srcs[json_cardname]
-
-                # look for a normal rarity version, in a set we can use
-                idx = 0
-                card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
-                while (idx < len(jcards)
-                       and (card.rarity == utils.rarity_special_marker 
-                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
-                    idx += 1
-                    if idx < len(jcards):
-                        card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
-                # if there isn't one, settle with index 0
-                if idx >= len(jcards):
-                    idx = 0
-                    card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
-                # we could go back and look for a card satisfying one of the criteria,
-                # but eh
-
-                if card.valid:
-                    valid += 1
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-                cards += [card]
-
-    # fall back to opening a normal encoded file
-    else:
-        if verbose:
-            print 'Opening encoded card file: ' + fname
-        with open(fname, 'rt') as f:
-            text = f.read()
-        for card_src in text.split(utils.cardsep):
-            if card_src:
-                card = cardlib.Card(card_src, fmt_ordered = fmt_ordered)
-                if card.valid:
-                    valid += 1
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-                cards += [card]
-
-    if verbose:
-        print (str(valid) + ' valid, ' + str(invalid) + ' invalid, ' 
-               + str(unparsed) + ' failed to parse.')
-
-    good_count = 0
-    bad_count = 0
-    for card in cards:
-        if not card.parsed and not card.text.text:
-            bad_count += 1
-        elif len(card.name) > 50 or len(card.rarity) > 3:
-            bad_count += 1
-        else:
-            good_count += 1
-        if good_count + bad_count > 15: 
-            break
-    # random heuristic
-    if bad_count > 10:
-        print 'WARNING: Saw a bunch of unparsed cards:'
-        print '         If this is a legacy format, try rerunning with "-e old" or "-e norarity"'
 
     if creativity:
         cbow = CBOW()

encode.py

@@ -10,15 +10,6 @@ import utils
 import jdecode
 import cardlib
 
-def exclude_sets(cardset):
-    return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
-
-def exclude_types(cardtype):
-    return cardtype in ['conspiracy']
-
-def exclude_layouts(layout):
-    return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
-
 def main(fname, oname = None, verbose = True, encoding = 'std', 
          nolinetrans = False, randomize = False, nolabel = False, stable = False):
     fmt_ordered = cardlib.fmt_ordered_default
@@ -67,76 +58,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std',
         if not line_transformations:
             print '  NOT using line reordering transformations'
 
-    cards = []
+    cards = jdecode.mtg_open_file(fname, verbose=verbose, linetrans=line_transformations)
-    valid = 0
-    skipped = 0
-    invalid = 0
-    unparsed = 0
-
-    if fname[-5:] == '.json':
-        if verbose:
-            print 'This looks like a json file: ' + fname
-        json_srcs = jdecode.mtg_open_json(fname, verbose)
-        # don't worry we randomize later
-        for json_cardname in sorted(json_srcs):
-            if len(json_srcs[json_cardname]) > 0:
-                jcards = json_srcs[json_cardname]
-
-                # look for a normal rarity version, in a set we can use
-                idx = 0
-                card = cardlib.Card(jcards[idx], linetrans = line_transformations)
-                while (idx < len(jcards)
-                       and (card.rarity == utils.rarity_special_marker 
-                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
-                    idx += 1
-                    if idx < len(jcards):
-                        card = cardlib.Card(jcards[idx], linetrans = line_transformations)
-                # if there isn't one, settle with index 0
-                if idx >= len(jcards):
-                    idx = 0
-                    card = cardlib.Card(jcards[idx], linetrans = line_transformations)
-                # we could go back and look for a card satisfying one of the criteria,
-                # but eh
-
-                skip = False
-                if (exclude_sets(jcards[idx][utils.json_field_set_name])
-                    or exclude_layouts(jcards[idx]['layout'])):
-                    skip = True                    
-                for cardtype in card.types:
-                    if exclude_types(cardtype):
-                        skip = True
-                if skip:
-                    skipped += 1
-                    continue
-                
-                if card.valid:
-                    valid += 1
-                    cards += [card]
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-
-    # fall back to opening a normal encoded file
-    else:
-        if verbose:
-            print 'Opening encoded card file: ' + fname
-        with open(fname, 'rt') as f:
-            text = f.read()
-        for card_src in text.split(utils.cardsep):
-            if card_src:
-                card = cardlib.Card(card_src)
-                if card.valid:
-                    valid += 1
-                    cards += [card]
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-
-    if verbose:
-        print (str(valid) + ' valid, ' + str(skipped) + ' skipped, ' 
-               + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
 
     # This should give a random but consistent ordering, to make comparing changes
     # between the output of different versions easier.

lib/jdecode.py

@@ -1,6 +1,7 @@
 import json
 
-import config
+import utils
+import cardlib
 
 def mtg_open_json(fname, verbose = False):
 
@@ -16,7 +17,7 @@ def mtg_open_json(fname, verbose = False):
         setname = set['name']
         
         for card in set['cards']:
-            card[config.json_field_set_name] = setname
+            card[utils.json_field_set_name] = setname
 
             cardnumber = None
             if 'number' in card:
@@ -48,7 +49,7 @@ def mtg_open_json(fname, verbose = False):
         if aside_uid in asides:
             # the second check handles the brothers yamazaki edge case
             if not asides[aside_uid]['name'] == bsides[uid]['name']:
-                asides[aside_uid][config.json_field_bside] = bsides[uid]
+                asides[aside_uid][utils.json_field_bside] = bsides[uid]
         else:
             pass
             # this exposes some coldsnap theme deck bsides that aren't
@@ -59,3 +60,109 @@ def mtg_open_json(fname, verbose = False):
     if verbose:
         print 'Opened ' + str(len(allcards)) + ' uniquely named cards.'
     return allcards
+
+# filters to ignore some undesirable cards, only used when opening json
+def default_exclude_sets(cardset):
+    return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
+
+def default_exclude_types(cardtype):
+    return cardtype in ['conspiracy']
+
+def default_exclude_layouts(layout):
+    return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
+
+# centralized logic for opening files of cards, either encoded or json
+def mtg_open_file(fname, verbose = False,
+                  linetrans = True, fmt_ordered = cardlib.fmt_ordered_default,
+                  exclude_sets = default_exclude_sets,
+                  exclude_types = default_exclude_types,
+                  exclude_layouts = default_exclude_layouts):
+
+    cards = []
+    valid = 0
+    skipped = 0
+    invalid = 0
+    unparsed = 0
+
+    if fname[-5:] == '.json':
+        if verbose:
+            print 'This looks like a json file: ' + fname
+        json_srcs = mtg_open_json(fname, verbose)
+        # sorted for stability
+        for json_cardname in sorted(json_srcs):
+            if len(json_srcs[json_cardname]) > 0:
+                jcards = json_srcs[json_cardname]
+
+                # look for a normal rarity version, in a set we can use
+                idx = 0
+                card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                while (idx < len(jcards)
+                       and (card.rarity == utils.rarity_special_marker
+                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
+                    idx += 1
+                    if idx < len(jcards):
+                        card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                # if there isn't one, settle with index 0
+                if idx >= len(jcards):
+                    idx = 0
+                    card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                # we could go back and look for a card satisfying one of the criteria,
+                # but eh
+
+                skip = False
+                if (exclude_sets(jcards[idx][utils.json_field_set_name])
+                    or exclude_layouts(jcards[idx]['layout'])):
+                    skip = True
+                for cardtype in card.types:
+                    if exclude_types(cardtype):
+                        skip = True
+                if skip:
+                    skipped += 1
+                    continue
+
+                if card.valid:
+                    valid += 1
+                    cards += [card]
+                elif card.parsed:
+                    invalid += 1
+                else:
+                    unparsed += 1
+
+    # fall back to opening a normal encoded file
+    else:
+        if verbose:
+            print 'Opening encoded card file: ' + fname
+        with open(fname, 'rt') as f:
+            text = f.read()
+        for card_src in text.split(utils.cardsep):
+            if card_src:
+                card = cardlib.Card(card_src, fmt_ordered=fmt_ordered)
+                if card.valid:
+                    valid += 1
+                    cards += [card]
+                elif card.parsed:
+                    invalid += 1
+                else:
+                    unparsed += 1
+
+    if verbose:
+        print (str(valid) + ' valid, ' + str(skipped) + ' skipped, '
+               + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
+
+    good_count = 0
+    bad_count = 0
+    for card in cards:
+        if not card.parsed and not card.text.text:
+            bad_count += 1
+        elif len(card.name) > 50 or len(card.rarity) > 3:
+            bad_count += 1
+        else:
+            good_count += 1
+        if good_count + bad_count > 15:
+            break
+    # random heuristic
+    if bad_count > 10:
+        print 'WARNING: Saw a bunch of unparsed cards:'
+        print '         Is this a legacy format, you may need to specify the field order.'
+
+    return cards