From b3860eb9249b8a163dbd5cf1542edf67c6fe7831 Mon Sep 17 00:00:00 2001 From: Bill Zorn Date: Fri, 6 Nov 2015 20:50:29 -0800 Subject: [PATCH] moved general file opening logic to a central location --- decode.py | 74 +------------------------------- encode.py | 80 +--------------------------------- lib/jdecode.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 155 deletions(-) diff --git a/decode.py b/decode.py index 3228110..6f53da0 100755 --- a/decode.py +++ b/decode.py @@ -43,79 +43,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std', else: raise ValueError('encode.py: unknown encoding: ' + encoding) - cards = [] - valid = 0 - invalid = 0 - unparsed = 0 - - if fname[-5:] == '.json': - if verbose: - print 'This looks like a json file: ' + fname - json_srcs = jdecode.mtg_open_json(fname, verbose) - for json_cardname in sorted(json_srcs): - if len(json_srcs[json_cardname]) > 0: - jcards = json_srcs[json_cardname] - - # look for a normal rarity version, in a set we can use - idx = 0 - card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered) - while (idx < len(jcards) - and (card.rarity == utils.rarity_special_marker - or exclude_sets(jcards[idx][utils.json_field_set_name]))): - idx += 1 - if idx < len(jcards): - card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered) - # if there isn't one, settle with index 0 - if idx >= len(jcards): - idx = 0 - card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered) - # we could go back and look for a card satisfying one of the criteria, - # but eh - - if card.valid: - valid += 1 - elif card.parsed: - invalid += 1 - else: - unparsed += 1 - cards += [card] - - # fall back to opening a normal encoded file - else: - if verbose: - print 'Opening encoded card file: ' + fname - with open(fname, 'rt') as f: - text = f.read() - for card_src in text.split(utils.cardsep): - if card_src: - card = cardlib.Card(card_src, fmt_ordered = fmt_ordered) - if card.valid: - valid += 1 - elif card.parsed: - invalid += 1 - else: - unparsed += 1 - cards += [card] - - if verbose: - print (str(valid) + ' valid, ' + str(invalid) + ' invalid, ' - + str(unparsed) + ' failed to parse.') - - good_count = 0 - bad_count = 0 - for card in cards: - if not card.parsed and not card.text.text: - bad_count += 1 - elif len(card.name) > 50 or len(card.rarity) > 3: - bad_count += 1 - else: - good_count += 1 - if good_count + bad_count > 15: - break - # random heuristic - if bad_count > 10: - print 'WARNING: Saw a bunch of unparsed cards:' - print ' If this is a legacy format, try rerunning with "-e old" or "-e norarity"' + cards = jdecode.mtg_open_file(fname, verbose=verbose, fmt_ordered=fmt_ordered) if creativity: cbow = CBOW() diff --git a/encode.py b/encode.py index bdc0326..e2996f8 100755 --- a/encode.py +++ b/encode.py @@ -10,15 +10,6 @@ import utils import jdecode import cardlib -def exclude_sets(cardset): - return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration' - -def exclude_types(cardtype): - return cardtype in ['conspiracy'] - -def exclude_layouts(layout): - return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard'] - def main(fname, oname = None, verbose = True, encoding = 'std', nolinetrans = False, randomize = False, nolabel = False, stable = False): fmt_ordered = cardlib.fmt_ordered_default @@ -67,76 +58,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std', if not line_transformations: print ' NOT using line reordering transformations' - cards = [] - valid = 0 - skipped = 0 - invalid = 0 - unparsed = 0 - - if fname[-5:] == '.json': - if verbose: - print 'This looks like a json file: ' + fname - json_srcs = jdecode.mtg_open_json(fname, verbose) - # don't worry we randomize later - for json_cardname in sorted(json_srcs): - if len(json_srcs[json_cardname]) > 0: - jcards = json_srcs[json_cardname] - - # look for a normal rarity version, in a set we can use - idx = 0 - card = cardlib.Card(jcards[idx], linetrans = line_transformations) - while (idx < len(jcards) - and (card.rarity == utils.rarity_special_marker - or exclude_sets(jcards[idx][utils.json_field_set_name]))): - idx += 1 - if idx < len(jcards): - card = cardlib.Card(jcards[idx], linetrans = line_transformations) - # if there isn't one, settle with index 0 - if idx >= len(jcards): - idx = 0 - card = cardlib.Card(jcards[idx], linetrans = line_transformations) - # we could go back and look for a card satisfying one of the criteria, - # but eh - - skip = False - if (exclude_sets(jcards[idx][utils.json_field_set_name]) - or exclude_layouts(jcards[idx]['layout'])): - skip = True - for cardtype in card.types: - if exclude_types(cardtype): - skip = True - if skip: - skipped += 1 - continue - - if card.valid: - valid += 1 - cards += [card] - elif card.parsed: - invalid += 1 - else: - unparsed += 1 - - # fall back to opening a normal encoded file - else: - if verbose: - print 'Opening encoded card file: ' + fname - with open(fname, 'rt') as f: - text = f.read() - for card_src in text.split(utils.cardsep): - if card_src: - card = cardlib.Card(card_src) - if card.valid: - valid += 1 - cards += [card] - elif card.parsed: - invalid += 1 - else: - unparsed += 1 - - if verbose: - print (str(valid) + ' valid, ' + str(skipped) + ' skipped, ' - + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.') + cards = jdecode.mtg_open_file(fname, verbose=verbose, linetrans=line_transformations) # This should give a random but consistent ordering, to make comparing changes # between the output of different versions easier. diff --git a/lib/jdecode.py b/lib/jdecode.py index ca66f86..fa2d3f7 100644 --- a/lib/jdecode.py +++ b/lib/jdecode.py @@ -1,6 +1,7 @@ import json -import config +import utils +import cardlib def mtg_open_json(fname, verbose = False): @@ -16,7 +17,7 @@ def mtg_open_json(fname, verbose = False): setname = set['name'] for card in set['cards']: - card[config.json_field_set_name] = setname + card[utils.json_field_set_name] = setname cardnumber = None if 'number' in card: @@ -48,7 +49,7 @@ def mtg_open_json(fname, verbose = False): if aside_uid in asides: # the second check handles the brothers yamazaki edge case if not asides[aside_uid]['name'] == bsides[uid]['name']: - asides[aside_uid][config.json_field_bside] = bsides[uid] + asides[aside_uid][utils.json_field_bside] = bsides[uid] else: pass # this exposes some coldsnap theme deck bsides that aren't @@ -59,3 +60,109 @@ def mtg_open_json(fname, verbose = False): if verbose: print 'Opened ' + str(len(allcards)) + ' uniquely named cards.' return allcards + +# filters to ignore some undesirable cards, only used when opening json +def default_exclude_sets(cardset): + return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration' + +def default_exclude_types(cardtype): + return cardtype in ['conspiracy'] + +def default_exclude_layouts(layout): + return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard'] + +# centralized logic for opening files of cards, either encoded or json +def mtg_open_file(fname, verbose = False, + linetrans = True, fmt_ordered = cardlib.fmt_ordered_default, + exclude_sets = default_exclude_sets, + exclude_types = default_exclude_types, + exclude_layouts = default_exclude_layouts): + + cards = [] + valid = 0 + skipped = 0 + invalid = 0 + unparsed = 0 + + if fname[-5:] == '.json': + if verbose: + print 'This looks like a json file: ' + fname + json_srcs = mtg_open_json(fname, verbose) + # sorted for stability + for json_cardname in sorted(json_srcs): + if len(json_srcs[json_cardname]) > 0: + jcards = json_srcs[json_cardname] + + # look for a normal rarity version, in a set we can use + idx = 0 + card = cardlib.Card(jcards[idx], linetrans=linetrans) + while (idx < len(jcards) + and (card.rarity == utils.rarity_special_marker + or exclude_sets(jcards[idx][utils.json_field_set_name]))): + idx += 1 + if idx < len(jcards): + card = cardlib.Card(jcards[idx], linetrans=linetrans) + # if there isn't one, settle with index 0 + if idx >= len(jcards): + idx = 0 + card = cardlib.Card(jcards[idx], linetrans=linetrans) + # we could go back and look for a card satisfying one of the criteria, + # but eh + + skip = False + if (exclude_sets(jcards[idx][utils.json_field_set_name]) + or exclude_layouts(jcards[idx]['layout'])): + skip = True + for cardtype in card.types: + if exclude_types(cardtype): + skip = True + if skip: + skipped += 1 + continue + + if card.valid: + valid += 1 + cards += [card] + elif card.parsed: + invalid += 1 + else: + unparsed += 1 + + # fall back to opening a normal encoded file + else: + if verbose: + print 'Opening encoded card file: ' + fname + with open(fname, 'rt') as f: + text = f.read() + for card_src in text.split(utils.cardsep): + if card_src: + card = cardlib.Card(card_src, fmt_ordered=fmt_ordered) + if card.valid: + valid += 1 + cards += [card] + elif card.parsed: + invalid += 1 + else: + unparsed += 1 + + if verbose: + print (str(valid) + ' valid, ' + str(skipped) + ' skipped, ' + + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.') + + good_count = 0 + bad_count = 0 + for card in cards: + if not card.parsed and not card.text.text: + bad_count += 1 + elif len(card.name) > 50 or len(card.rarity) > 3: + bad_count += 1 + else: + good_count += 1 + if good_count + bad_count > 15: + break + # random heuristic + if bad_count > 10: + print 'WARNING: Saw a bunch of unparsed cards:' + print ' Is this a legacy format, you may need to specify the field order.' + + return cards