moved general file opening logic to a central location
This commit is contained in:
Bill Zorn
parent
1fd9da4b9f
commit
b3860eb924
3 changed files with 112 additions and 155 deletions
74
decode.py
74
decode.py
|
|
@ -43,79 +43,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std',
|
|||
else:
|
||||
raise ValueError('encode.py: unknown encoding: ' + encoding)
|
||||
|
||||
cards = []
|
||||
valid = 0
|
||||
invalid = 0
|
||||
unparsed = 0
|
||||
|
||||
if fname[-5:] == '.json':
|
||||
if verbose:
|
||||
print 'This looks like a json file: ' + fname
|
||||
json_srcs = jdecode.mtg_open_json(fname, verbose)
|
||||
for json_cardname in sorted(json_srcs):
|
||||
if len(json_srcs[json_cardname]) > 0:
|
||||
jcards = json_srcs[json_cardname]
|
||||
|
||||
# look for a normal rarity version, in a set we can use
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
|
||||
while (idx < len(jcards)
|
||||
and (card.rarity == utils.rarity_special_marker
|
||||
or exclude_sets(jcards[idx][utils.json_field_set_name]))):
|
||||
idx += 1
|
||||
if idx < len(jcards):
|
||||
card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
|
||||
# if there isn't one, settle with index 0
|
||||
if idx >= len(jcards):
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], fmt_ordered = fmt_ordered)
|
||||
# we could go back and look for a card satisfying one of the criteria,
|
||||
# but eh
|
||||
|
||||
if card.valid:
|
||||
valid += 1
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
cards += [card]
|
||||
|
||||
# fall back to opening a normal encoded file
|
||||
else:
|
||||
if verbose:
|
||||
print 'Opening encoded card file: ' + fname
|
||||
with open(fname, 'rt') as f:
|
||||
text = f.read()
|
||||
for card_src in text.split(utils.cardsep):
|
||||
if card_src:
|
||||
card = cardlib.Card(card_src, fmt_ordered = fmt_ordered)
|
||||
if card.valid:
|
||||
valid += 1
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
cards += [card]
|
||||
|
||||
if verbose:
|
||||
print (str(valid) + ' valid, ' + str(invalid) + ' invalid, '
|
||||
+ str(unparsed) + ' failed to parse.')
|
||||
|
||||
good_count = 0
|
||||
bad_count = 0
|
||||
for card in cards:
|
||||
if not card.parsed and not card.text.text:
|
||||
bad_count += 1
|
||||
elif len(card.name) > 50 or len(card.rarity) > 3:
|
||||
bad_count += 1
|
||||
else:
|
||||
good_count += 1
|
||||
if good_count + bad_count > 15:
|
||||
break
|
||||
# random heuristic
|
||||
if bad_count > 10:
|
||||
print 'WARNING: Saw a bunch of unparsed cards:'
|
||||
print ' If this is a legacy format, try rerunning with "-e old" or "-e norarity"'
|
||||
cards = jdecode.mtg_open_file(fname, verbose=verbose, fmt_ordered=fmt_ordered)
|
||||
|
||||
if creativity:
|
||||
cbow = CBOW()
|
||||
|
|
|
|||
80
encode.py
80
encode.py
|
|
@ -10,15 +10,6 @@ import utils
|
|||
import jdecode
|
||||
import cardlib
|
||||
|
||||
def exclude_sets(cardset):
|
||||
return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
|
||||
|
||||
def exclude_types(cardtype):
|
||||
return cardtype in ['conspiracy']
|
||||
|
||||
def exclude_layouts(layout):
|
||||
return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
|
||||
|
||||
def main(fname, oname = None, verbose = True, encoding = 'std',
|
||||
nolinetrans = False, randomize = False, nolabel = False, stable = False):
|
||||
fmt_ordered = cardlib.fmt_ordered_default
|
||||
|
|
@ -67,76 +58,7 @@ def main(fname, oname = None, verbose = True, encoding = 'std',
|
|||
if not line_transformations:
|
||||
print ' NOT using line reordering transformations'
|
||||
|
||||
cards = []
|
||||
valid = 0
|
||||
skipped = 0
|
||||
invalid = 0
|
||||
unparsed = 0
|
||||
|
||||
if fname[-5:] == '.json':
|
||||
if verbose:
|
||||
print 'This looks like a json file: ' + fname
|
||||
json_srcs = jdecode.mtg_open_json(fname, verbose)
|
||||
# don't worry we randomize later
|
||||
for json_cardname in sorted(json_srcs):
|
||||
if len(json_srcs[json_cardname]) > 0:
|
||||
jcards = json_srcs[json_cardname]
|
||||
|
||||
# look for a normal rarity version, in a set we can use
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], linetrans = line_transformations)
|
||||
while (idx < len(jcards)
|
||||
and (card.rarity == utils.rarity_special_marker
|
||||
or exclude_sets(jcards[idx][utils.json_field_set_name]))):
|
||||
idx += 1
|
||||
if idx < len(jcards):
|
||||
card = cardlib.Card(jcards[idx], linetrans = line_transformations)
|
||||
# if there isn't one, settle with index 0
|
||||
if idx >= len(jcards):
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], linetrans = line_transformations)
|
||||
# we could go back and look for a card satisfying one of the criteria,
|
||||
# but eh
|
||||
|
||||
skip = False
|
||||
if (exclude_sets(jcards[idx][utils.json_field_set_name])
|
||||
or exclude_layouts(jcards[idx]['layout'])):
|
||||
skip = True
|
||||
for cardtype in card.types:
|
||||
if exclude_types(cardtype):
|
||||
skip = True
|
||||
if skip:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if card.valid:
|
||||
valid += 1
|
||||
cards += [card]
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
|
||||
# fall back to opening a normal encoded file
|
||||
else:
|
||||
if verbose:
|
||||
print 'Opening encoded card file: ' + fname
|
||||
with open(fname, 'rt') as f:
|
||||
text = f.read()
|
||||
for card_src in text.split(utils.cardsep):
|
||||
if card_src:
|
||||
card = cardlib.Card(card_src)
|
||||
if card.valid:
|
||||
valid += 1
|
||||
cards += [card]
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
|
||||
if verbose:
|
||||
print (str(valid) + ' valid, ' + str(skipped) + ' skipped, '
|
||||
+ str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
|
||||
cards = jdecode.mtg_open_file(fname, verbose=verbose, linetrans=line_transformations)
|
||||
|
||||
# This should give a random but consistent ordering, to make comparing changes
|
||||
# between the output of different versions easier.
|
||||
|
|
|
|||
113
lib/jdecode.py
113
lib/jdecode.py
|
|
@ -1,6 +1,7 @@
|
|||
import json
|
||||
|
||||
import config
|
||||
import utils
|
||||
import cardlib
|
||||
|
||||
def mtg_open_json(fname, verbose = False):
|
||||
|
||||
|
|
@ -16,7 +17,7 @@ def mtg_open_json(fname, verbose = False):
|
|||
setname = set['name']
|
||||
|
||||
for card in set['cards']:
|
||||
card[config.json_field_set_name] = setname
|
||||
card[utils.json_field_set_name] = setname
|
||||
|
||||
cardnumber = None
|
||||
if 'number' in card:
|
||||
|
|
@ -48,7 +49,7 @@ def mtg_open_json(fname, verbose = False):
|
|||
if aside_uid in asides:
|
||||
# the second check handles the brothers yamazaki edge case
|
||||
if not asides[aside_uid]['name'] == bsides[uid]['name']:
|
||||
asides[aside_uid][config.json_field_bside] = bsides[uid]
|
||||
asides[aside_uid][utils.json_field_bside] = bsides[uid]
|
||||
else:
|
||||
pass
|
||||
# this exposes some coldsnap theme deck bsides that aren't
|
||||
|
|
@ -59,3 +60,109 @@ def mtg_open_json(fname, verbose = False):
|
|||
if verbose:
|
||||
print 'Opened ' + str(len(allcards)) + ' uniquely named cards.'
|
||||
return allcards
|
||||
|
||||
# filters to ignore some undesirable cards, only used when opening json
|
||||
def default_exclude_sets(cardset):
|
||||
return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
|
||||
|
||||
def default_exclude_types(cardtype):
|
||||
return cardtype in ['conspiracy']
|
||||
|
||||
def default_exclude_layouts(layout):
|
||||
return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
|
||||
|
||||
# centralized logic for opening files of cards, either encoded or json
|
||||
def mtg_open_file(fname, verbose = False,
|
||||
linetrans = True, fmt_ordered = cardlib.fmt_ordered_default,
|
||||
exclude_sets = default_exclude_sets,
|
||||
exclude_types = default_exclude_types,
|
||||
exclude_layouts = default_exclude_layouts):
|
||||
|
||||
cards = []
|
||||
valid = 0
|
||||
skipped = 0
|
||||
invalid = 0
|
||||
unparsed = 0
|
||||
|
||||
if fname[-5:] == '.json':
|
||||
if verbose:
|
||||
print 'This looks like a json file: ' + fname
|
||||
json_srcs = mtg_open_json(fname, verbose)
|
||||
# sorted for stability
|
||||
for json_cardname in sorted(json_srcs):
|
||||
if len(json_srcs[json_cardname]) > 0:
|
||||
jcards = json_srcs[json_cardname]
|
||||
|
||||
# look for a normal rarity version, in a set we can use
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], linetrans=linetrans)
|
||||
while (idx < len(jcards)
|
||||
and (card.rarity == utils.rarity_special_marker
|
||||
or exclude_sets(jcards[idx][utils.json_field_set_name]))):
|
||||
idx += 1
|
||||
if idx < len(jcards):
|
||||
card = cardlib.Card(jcards[idx], linetrans=linetrans)
|
||||
# if there isn't one, settle with index 0
|
||||
if idx >= len(jcards):
|
||||
idx = 0
|
||||
card = cardlib.Card(jcards[idx], linetrans=linetrans)
|
||||
# we could go back and look for a card satisfying one of the criteria,
|
||||
# but eh
|
||||
|
||||
skip = False
|
||||
if (exclude_sets(jcards[idx][utils.json_field_set_name])
|
||||
or exclude_layouts(jcards[idx]['layout'])):
|
||||
skip = True
|
||||
for cardtype in card.types:
|
||||
if exclude_types(cardtype):
|
||||
skip = True
|
||||
if skip:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if card.valid:
|
||||
valid += 1
|
||||
cards += [card]
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
|
||||
# fall back to opening a normal encoded file
|
||||
else:
|
||||
if verbose:
|
||||
print 'Opening encoded card file: ' + fname
|
||||
with open(fname, 'rt') as f:
|
||||
text = f.read()
|
||||
for card_src in text.split(utils.cardsep):
|
||||
if card_src:
|
||||
card = cardlib.Card(card_src, fmt_ordered=fmt_ordered)
|
||||
if card.valid:
|
||||
valid += 1
|
||||
cards += [card]
|
||||
elif card.parsed:
|
||||
invalid += 1
|
||||
else:
|
||||
unparsed += 1
|
||||
|
||||
if verbose:
|
||||
print (str(valid) + ' valid, ' + str(skipped) + ' skipped, '
|
||||
+ str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
|
||||
|
||||
good_count = 0
|
||||
bad_count = 0
|
||||
for card in cards:
|
||||
if not card.parsed and not card.text.text:
|
||||
bad_count += 1
|
||||
elif len(card.name) > 50 or len(card.rarity) > 3:
|
||||
bad_count += 1
|
||||
else:
|
||||
good_count += 1
|
||||
if good_count + bad_count > 15:
|
||||
break
|
||||
# random heuristic
|
||||
if bad_count > 10:
|
||||
print 'WARNING: Saw a bunch of unparsed cards:'
|
||||
print ' Is this a legacy format, you may need to specify the field order.'
|
||||
|
||||
return cards
|
||||
|
|
|
|||
Loading…
Reference in a new issue