updating to the newest version of mtgencode and finishing html support

decode.py

@@ -15,95 +15,35 @@ from namediff import Namediff
 def exclude_sets(cardset):
     return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
 
-def main(fname, oname = None, verbose = True, 
+def main(fname, oname = None, verbose = True, encoding = 'std',
          gatherer = False, for_forum = False, for_mse = False,
-         creativity = False, norarity = False, vdump = False, for_html = False):
-    cards = []
-    valid = 0
-    invalid = 0
-    unparsed = 0
+         creativity = False, vdump = False, for_html = False):
 
-    if norarity:
-        decode_fields = [
-            cardlib.field_name,
-            cardlib.field_supertypes,
-            cardlib.field_types,
-            cardlib.field_loyalty,
-            cardlib.field_subtypes,
-            #cardlib.field_rarity,
-            cardlib.field_pt,
-            cardlib.field_cost,
-            cardlib.field_text,
-        ]
+    fmt_ordered = cardlib.fmt_ordered_default
+
+    if encoding in ['std']:
+        pass
+    elif encoding in ['named']:
+        fmt_ordered = cardlib.fmt_ordered_named
+    elif encoding in ['noname']:
+        fmt_ordered = cardlib.fmt_ordered_noname
+    elif encoding in ['rfields']:
+        pass
+    elif encoding in ['old']:
+        fmt_ordered = cardlib.fmt_ordered_old
+    elif encoding in ['norarity']:
+        fmt_ordered = cardlib.fmt_ordered_norarity
+    elif encoding in ['vec']:
+        pass
+    elif encoding in ['custom']:
+        ## put custom format decisions here ##########################
+        
+        ## end of custom format ######################################
+        pass
     else:
-        decode_fields = cardlib.fmt_ordered_default
+        raise ValueError('encode.py: unknown encoding: ' + encoding)
 
-    if fname[-5:] == '.json':
-        if verbose:
-            print 'This looks like a json file: ' + fname
-        json_srcs = jdecode.mtg_open_json(fname, verbose)
-        for json_cardname in sorted(json_srcs):
-            if len(json_srcs[json_cardname]) > 0:
-                jcards = json_srcs[json_cardname]
-
-                # look for a normal rarity version, in a set we can use
-                idx = 0
-                card = cardlib.Card(jcards[idx], fmt_ordered = decode_fields)
-                while (idx < len(jcards)
-                       and (card.rarity == utils.rarity_special_marker 
-                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
-                    idx += 1
-                    if idx < len(jcards):
-                        card = cardlib.Card(jcards[idx], fmt_ordered = decode_fields)
-                # if there isn't one, settle with index 0
-                if idx >= len(jcards):
-                    idx = 0
-                    card = cardlib.Card(jcards[idx], fmt_ordered = decode_fields)
-                # we could go back and look for a card satisfying one of the criteria,
-                # but eh
-
-                if card.valid:
-                    valid += 1
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-                cards += [card]
-
-    # fall back to opening a normal encoded file
-    else:
-        if verbose:
-            print 'Opening encoded card file: ' + fname
-        with open(fname, 'rt') as f:
-            text = f.read()
-        for card_src in text.split(utils.cardsep):
-            if card_src:
-                card = cardlib.Card(card_src, fmt_ordered = decode_fields)
-                if card.valid:
-                    valid += 1
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-                cards += [card]
-
-    if verbose:
-        print (str(valid) + ' valid, ' + str(invalid) + ' invalid, ' 
-               + str(unparsed) + ' failed to parse.')
-
-    good_count = 0
-    bad_count = 0
-    for card in cards:
-        if not card.parsed and not card.text.text:
-            bad_count += 1
-        else:
-            good_count += 1
-        if good_count + bad_count > 15: 
-            break
-    # random heuristic
-    if bad_count > 10:
-        print 'Saw a bunch of unparsed cards with no text:'
-        print 'If this is a legacy format, try rerunning with --norarity'
+    cards = jdecode.mtg_open_file(fname, verbose=verbose, fmt_ordered=fmt_ordered)
 
     if creativity:
         cbow = CBOW()
@@ -116,7 +56,7 @@ def main(fname, oname = None, verbose = True,
 
         if for_html:
             # have to preapend html info
-            writer.write(utils.html_preapend)
+            writer.write(utils.html_prepend)
 
         for card in cards:
             if for_mse:
@@ -128,7 +68,7 @@ def main(fname, oname = None, verbose = True,
                     fstring += 'raw:\n' + card.raw + '\n'
                 fstring += '\n'
                 fstring += card.format(gatherer = gatherer, for_forum = for_forum,
-                                       vdump = vdump, for_html = for_html)
+                                       vdump = vdump)
                 fstring = fstring.replace('<', '(').replace('>', ')')
                 writer.write(('\n' + fstring[:-1]).replace('\n', '\n\t\t'))
             else:
@@ -160,12 +100,15 @@ def main(fname, oname = None, verbose = True,
         if for_mse:
             # more formatting info
             writer.write('version control:\n\ttype: none\napprentice code: ')
-            
         if for_html:
             # closing the html file
-            writer.write(utils.html_postapend)
+            writer.write(utils.html_append)
 
     if oname:
+        if for_html:
+            print oname
+            # if ('.html' != oname[-])
+            #     oname += '.html'
         if verbose:
             print 'Writing output to: ' + oname
         with open(oname, 'w') as ofile:
@@ -186,8 +129,6 @@ def main(fname, oname = None, verbose = True,
                         print 'Made an MSE set file called ' + oname + '.mse-set.'
                     # The set file is useless outside the .mse-set, delete it.
                     os.remove('set') 
-        if for_html:
-                ## not sure what to put here
     else:
         writecards(sys.stdout)
         sys.stdout.flush()
@@ -201,6 +142,10 @@ if __name__ == '__main__':
                         help='encoded card file or json corpus to encode')
     parser.add_argument('outfile', nargs='?', default=None,
                         help='output file, defaults to stdout')
+    parser.add_argument('-e', '--encoding', default='std', choices=utils.formats,
+                        #help='{' + ','.join(formats) + '}',
+                        help='encoding format to use',
+    )
     parser.add_argument('-g', '--gatherer', action='store_true',
                         help='emulate Gatherer visual spoiler')
     parser.add_argument('-f', '--forum', action='store_true',
@@ -209,16 +154,13 @@ if __name__ == '__main__':
                         help='use CBOW fuzzy matching to check creativity of cards')
     parser.add_argument('-d', '--dump', action='store_true',
                         help='dump out lots of information about invalid cards')
-    parser.add_argument('--norarity', action='store_true',
-                        help='the card format has no rarity field; use for legacy input')
     parser.add_argument('-v', '--verbose', action='store_true', 
                         help='verbose output')
     parser.add_argument('-mse', '--mse', action='store_true', 
                         help='use Magic Set Editor 2 encoding; will output as .mse-set file')
     parser.add_argument('-html', '--html', action='store_true', help='create a .html file with pretty forum formatting')
-    
     args = parser.parse_args()
-    main(args.infile, args.outfile, verbose = args.verbose, 
+    main(args.infile, args.outfile, verbose = args.verbose, encoding = args.encoding,
          gatherer = args.gatherer, for_forum = args.forum, for_mse = args.mse,
-         creativity = args.creativity, norarity = args.norarity, vdump = args.dump, for_html = args.for_html)
+         creativity = args.creativity, vdump = args.dump, for_html = args.html)
     exit(0)

encode.py

@@ -10,131 +10,55 @@ import utils
 import jdecode
 import cardlib
 
-def exclude_sets(cardset):
-    return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
-
-def exclude_types(cardtype):
-    return cardtype in ['conspiracy']
-
-def exclude_layouts(layout):
-    return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
-
-def main(fname, oname = None, verbose = True, dupes = 0, encoding = 'std', stable = False):
+def main(fname, oname = None, verbose = True, encoding = 'std', 
+         nolinetrans = False, randomize = False, nolabel = False, stable = False):
     fmt_ordered = cardlib.fmt_ordered_default
-    fmt_labeled = None
+    fmt_labeled = None if nolabel else cardlib.fmt_labeled_default
     fieldsep = utils.fieldsep
+    line_transformations = not nolinetrans
     randomize_fields = False
-    randomize_mana = False
+    randomize_mana = randomize
     initial_sep = True
     final_sep = True
 
     # set the properties of the encoding
-    if encoding in ['vec']:
+
+    if encoding in ['std']:
         pass
-    elif encoding in ['std']:
-        if dupes == 0:
-            dupes = 1
-    elif encoding in ['rmana']:
-        if dupes == 0:
-            dupes = 1
-        randomize_mana = True
-    elif encoding in ['rmana_dual']:
-        if dupes == 0:
-            dupes = 1
-        fmt_ordered = fmt_ordered + [cardlib.field_cost]
-        randomize_mana = True
+    elif encoding in ['named']:
+        fmt_ordered = cardlib.fmt_ordered_named
+    elif encoding in ['noname']:
+        fmt_ordered = cardlib.fmt_ordered_noname
     elif encoding in ['rfields']:
-        if dupes == 0:
-            dupes = 1
-        fmt_labeled = cardlib.fmt_labeled_default
         randomize_fields = True
-        #randomize_mana = True
         final_sep = False
+    elif encoding in ['old']:
+        fmt_ordered = cardlib.fmt_ordered_old
+    elif encoding in ['norarity']:
+        fmt_ordered = cardlib.fmt_ordered_norarity
+    elif encoding in ['vec']:
+        pass
+    elif encoding in ['custom']:
+        ## put custom format decisions here ##########################
+        
+        ## end of custom format ######################################
+        pass
     else:
         raise ValueError('encode.py: unknown encoding: ' + encoding)
 
-    if dupes <= 0:
-        dupes = 1 
-
     if verbose:
         print 'Preparing to encode:'
         print '  Using encoding ' + repr(encoding)
-        if dupes > 1:
-            print '  Duplicating each card ' + str(dupes) + ' times.'
         if stable:
             print '  NOT randomizing order of cards.'
+        if randomize_mana:
+            print '  Randomizing order of symobls in manacosts.'
+        if not fmt_labeled:
+            print '  NOT labeling fields for this run (may be harder to decode).'
+        if not line_transformations:
+            print '  NOT using line reordering transformations'
 
-
-    cards = []
-    valid = 0
-    skipped = 0
-    invalid = 0
-    unparsed = 0
-
-    if fname[-5:] == '.json':
-        if verbose:
-            print 'This looks like a json file: ' + fname
-        json_srcs = jdecode.mtg_open_json(fname, verbose)
-        # don't worry we randomize later
-        for json_cardname in sorted(json_srcs):
-            if len(json_srcs[json_cardname]) > 0:
-                jcards = json_srcs[json_cardname]
-
-                # look for a normal rarity version, in a set we can use
-                idx = 0
-                card = cardlib.Card(jcards[idx])
-                while (idx < len(jcards)
-                       and (card.rarity == utils.rarity_special_marker 
-                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
-                    idx += 1
-                    if idx < len(jcards):
-                        card = cardlib.Card(jcards[idx])
-                # if there isn't one, settle with index 0
-                if idx >= len(jcards):
-                    idx = 0
-                    card = cardlib.Card(jcards[idx])
-                # we could go back and look for a card satisfying one of the criteria,
-                # but eh
-
-                skip = False
-                if (exclude_sets(jcards[idx][utils.json_field_set_name])
-                    or exclude_layouts(jcards[idx]['layout'])):
-                    skip = True                    
-                for cardtype in card.types:
-                    if exclude_types(cardtype):
-                        skip = True
-                if skip:
-                    skipped += 1
-                    continue
-                
-                if card.valid:
-                    valid += 1
-                    cards += [card] * dupes
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-
-    # fall back to opening a normal encoded file
-    else:
-        if verbose:
-            print 'Opening encoded card file: ' + fname
-        with open(fname, 'rt') as f:
-            text = f.read()
-        for card_src in text.split(utils.cardsep):
-            if card_src:
-                card = cardlib.Card(card_src)
-                if card.valid:
-                    valid += 1
-                    cards += [card] * dupes
-                elif card.parsed:
-                    invalid += 1
-                else:
-                    unparsed += 1
-
-    if verbose:
-        print (str(valid) + ' valid, ' + str(skipped) + ' skipped, ' 
-               + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
+    cards = jdecode.mtg_open_file(fname, verbose=verbose, linetrans=line_transformations)
 
     # This should give a random but consistent ordering, to make comparing changes
     # between the output of different versions easier.
@@ -174,17 +98,23 @@ if __name__ == '__main__':
                         help='encoded card file or json corpus to encode')
     parser.add_argument('outfile', nargs='?', default=None,
                         help='output file, defaults to stdout')
-    parser.add_argument('-d', '--duplicate', metavar='N', type=int, default=0,
-                        help='number of times to duplicate each card')
-    parser.add_argument('-e', '--encoding', default='std',
-                        choices=['std', 'rmana', 'rmana_dual', 'rfields', 'vec'])
+    parser.add_argument('-e', '--encoding', default='std', choices=utils.formats,
+                        #help='{' + ','.join(formats) + '}',
+                        help='encoding format to use',
+    )
+    parser.add_argument('-r', '--randomize', action='store_true',
+                        help='randomize the order of symbols in mana costs')
+    parser.add_argument('--nolinetrans', action='store_true',
+                        help="don't reorder lines of card text")
+    parser.add_argument('--nolabel', action='store_true',
+                        help="don't label fields")
     parser.add_argument('-s', '--stable', action='store_true',
                         help="don't randomize the order of the cards")
     parser.add_argument('-v', '--verbose', action='store_true', 
                         help='verbose output')
     
     args = parser.parse_args()
-    main(args.infile, args.outfile, verbose = args.verbose, dupes = args.duplicate,
-         encoding = args.encoding, stable = args.stable)
+    main(args.infile, args.outfile, verbose = args.verbose, encoding = args.encoding, 
+         nolinetrans = args.nolinetrans, randomize = args.randomize, nolabel = args.nolabel, 
+         stable = args.stable)
     exit(0)
-

lib/cardlib.py

@@ -88,7 +88,8 @@ fieldnames = [
     field_text,
 ]
 
-fmt_ordered_default = [
+# legacy
+fmt_ordered_old = [
     field_name,
     field_supertypes,
     field_types,
@@ -99,6 +100,52 @@ fmt_ordered_default = [
     field_cost,
     field_text,
 ]
+fmt_ordered_norarity = [
+    field_name,
+    field_supertypes,
+    field_types,
+    field_loyalty,
+    field_subtypes,
+    field_pt,
+    field_cost,
+    field_text,
+]
+
+# standard
+fmt_ordered_default = [
+    field_types,
+    field_supertypes,
+    field_subtypes,
+    field_loyalty,
+    field_pt,
+    field_text,
+    field_cost,
+    field_rarity,
+    field_name,
+]
+
+# minor variations
+fmt_ordered_noname = [
+    field_types,
+    field_supertypes,
+    field_subtypes,
+    field_loyalty,
+    field_pt,
+    field_text,
+    field_cost,
+    field_rarity,
+]
+fmt_ordered_named = [
+    field_name,
+    field_types,
+    field_supertypes,
+    field_subtypes,
+    field_loyalty,
+    field_pt,
+    field_text,
+    field_cost,
+    field_rarity,
+]
 
 fmt_labeled_default = {
     field_name : field_label_name,
@@ -176,7 +223,7 @@ def fields_check_valid(fields):
 # releaseDate - string
 # starter - boolean
 
-def fields_from_json(src_json):
+def fields_from_json(src_json, linetrans = True):
     parsed = True
     valid = True
     fields = {}
@@ -254,6 +301,8 @@ def fields_from_json(src_json):
         text_val = transforms.text_pass_8_equip(text_val)
         text_val = transforms.text_pass_9_newlines(text_val)
         text_val = transforms.text_pass_10_symbols(text_val)
+        if linetrans:
+            text_val = transforms.text_pass_11_linetrans(text_val)
         text_val = utils.to_ascii(text_val)
         text_val = text_val.strip()
         mtext = Manatext(text_val, fmt = 'json')
@@ -340,9 +389,7 @@ def fields_from_format(src_text, fmt_ordered, fmt_labeled, fieldsep):
 class Card:
     '''card representation with data'''
 
-    def __init__(self, src, fmt_ordered = fmt_ordered_default, 
-                            fmt_labeled = fmt_labeled_default, 
-                            fieldsep = utils.fieldsep):
+    def __init__(self, src, fmt_ordered = fmt_ordered_default, fmt_labeled = fmt_labeled_default, fieldsep = utils.fieldsep, linetrans = True):
         # source fields, exactly one will be set
         self.json = None
         self.raw = None
@@ -378,8 +425,9 @@ class Card:
                 self.bside = Card(src[utils.json_field_bside],
                                   fmt_ordered = fmt_ordered,
                                   fmt_labeled = fmt_labeled,
-                                  fieldsep = fieldsep)
-            p_success, v_success, parsed_fields = fields_from_json(src)
+                                  fieldsep = fieldsep,
+                                  linetrans = linetrans)
+            p_success, v_success, parsed_fields = fields_from_json(src, linetrans = linetrans)
             self.parsed = p_success
             self.valid = v_success
             self.fields = parsed_fields
@@ -391,7 +439,8 @@ class Card:
                 self.bside = Card(utils.bsidesep.join(sides[1:]), 
                                   fmt_ordered = fmt_ordered,
                                   fmt_labeled = fmt_labeled,
-                                  fieldsep = fieldsep)
+                                  fieldsep = fieldsep,
+                                  linetrans = linetrans)
             p_success, v_success, parsed_fields = fields_from_format(sides[0], fmt_ordered, 
                                                                      fmt_labeled,  fieldsep)
             self.parsed = p_success
@@ -503,10 +552,7 @@ class Card:
     # the NN representation, use str() or format() for output intended for human
     # readers.
 
-    def encode(self, fmt_ordered = fmt_ordered_default,
-               fmt_labeled = None, fieldsep = utils.fieldsep,
-               randomize_fields = False, randomize_mana = False,
-               initial_sep = True, final_sep = True):
+    def encode(self, fmt_ordered = fmt_ordered_default, fmt_labeled = None, fieldsep = utils.fieldsep, randomize_fields = False, randomize_mana = False, initial_sep = True, final_sep = True):
         outfields = []
 
         for field in fmt_ordered:
@@ -608,7 +654,7 @@ class Card:
                 mtext = transforms.text_unpass_2_counters(mtext)
                 #mtext = transforms.text_unpass_3_uncast(mtext)
                 mtext = transforms.text_unpass_4_unary(mtext)
-                mtext = transforms.text_unpass_5_symbols(mtext, for_forum)
+                mtext = transforms.text_unpass_5_symbols(mtext, for_forum, for_html)
                 mtext = sentencecase(mtext)
                 mtext = transforms.text_unpass_6_cardname(mtext, cardname)
                 mtext = transforms.text_unpass_7_newlines(mtext)
@@ -638,25 +684,27 @@ class Card:
             outstr += '<div class="card-text">'
             cardname = self.__dict__[field_name]
             #cardname = transforms.name_unpass_1_dashes(self.__dict__[field_name])
-            outstr += "<h5>" + cardname + "</h5>"
+            if vdump and not cardname:
+                cardname = '_NONAME_'
+            outstr += cardname + ' '
+            
+            coststr = self.__dict__[field_cost].format(for_html = for_html)
+            if vdump or not coststr == '_NOCOST_':
+                outstr += coststr
+                outstr += '<br>'
+                
             if self.__dict__[field_rarity]:
                 if self.__dict__[field_rarity] in utils.json_rarity_unmap:
                     rarity = utils.json_rarity_unmap[self.__dict__[field_rarity]]
                 else:
                     rarity = self.__dict__[field_rarity]
-                outstr += ' (<b>' + rarity.lower() + '</b>)'
-            outstr += '\n'
+                outstr += ' (' + rarity.lower() + ') '
+            outstr += '\n<hr><b>'
 
-            # I need the simple formatting with '{'
-            coststr = self.__dict__[field_cost].format()
-            if vdump or not coststr == '_NOCOST_':
-                outstr += coststr.replace("/","-").replace("{",'<img src="~/mtgencode/Icons/' ).replace("}",'-mana;.png" >')
-                outstr += '\n'
-
-            outstr += ' <b>'.join(self.__dict__[field_supertypes] + self.__dict__[field_types])
+            outstr += ' '.join(self.__dict__[field_supertypes] + self.__dict__[field_types])
             if self.__dict__[field_subtypes]:
                 outstr += ' ' + utils.dash_marker + ' ' + ' '.join(self.__dict__[field_subtypes])
-            outstr += '</b>\n'
+            outstr += '</b><hr>\n'
             
             if self.__dict__[field_text].text:
                 mtext = self.__dict__[field_text].text
@@ -664,17 +712,17 @@ class Card:
                 #mtext = transforms.text_unpass_2_counters(mtext)
                 #mtext = transforms.text_unpass_3_uncast(mtext)
                 mtext = transforms.text_unpass_4_unary(mtext)
-                mtext = transforms.text_unpass_5_symbols(mtext, for_forum)
+                mtext = transforms.text_unpass_5_symbols(mtext,for_forum, for_html)
                 #mtext = transforms.text_unpass_6_cardname(mtext, cardname)
-                mtext = transforms.text_unpass_7_newlines(mtext)
+                mtext = transforms.text_unpass_7_newlines(mtext).replace("\n", "<br>")
                 #mtext = transforms.text_unpass_8_unicode(mtext)
                 newtext = Manatext('')
                 newtext.text = mtext
                 newtext.costs = self.__dict__[field_text].costs
-                outstr += newtext.format().replace("/","-").replace("{",'<img src="~/mtgencode/Icons/' ).replace("}",'-mana;.png" >') + '\n'
+                outstr += newtext.format(for_html = for_html) + '\n'
 
             if self.__dict__[field_pt]:
-                outstr += '(' + utils.from_unary(self.__dict__[field_pt]) + ')'
+                outstr += '<br>(' + utils.from_unary(self.__dict__[field_pt]) + ')<br>'
                 outstr += '\n'
 
             if self.__dict__[field_loyalty]:
@@ -687,7 +735,6 @@ class Card:
                 for idx, value in self.__dict__[field_other]:
                     outstr += '<' + str(idx) + '> ' + str(value)
                     outstr += '\n'
-        
         else:
             cardname = self.__dict__[field_name]
             #cardname = transforms.name_unpass_1_dashes(self.__dict__[field_name])
@@ -723,7 +770,7 @@ class Card:
                 #mtext = transforms.text_unpass_2_counters(mtext)
                 #mtext = transforms.text_unpass_3_uncast(mtext)
                 mtext = transforms.text_unpass_4_unary(mtext)
-                mtext = transforms.text_unpass_5_symbols(mtext, for_forum)
+                mtext = transforms.text_unpass_5_symbols(mtext, for_forum, for_html)
                 #mtext = transforms.text_unpass_6_cardname(mtext, cardname)
                 mtext = transforms.text_unpass_7_newlines(mtext)
                 #mtext = transforms.text_unpass_8_unicode(mtext)
@@ -749,11 +796,12 @@ class Card:
 
         if self.bside:
             if for_html:
-                outstr += "<hr>\n"
+                outstr += "<hr><hr>\n"
             else:
                 outstr += utils.dash_marker * 8 + '\n'
-            outstr += self.bside.format(gatherer = gatherer, for_forum = for_forum)
-
+            outstr += self.bside.format(gatherer = gatherer, for_forum = for_forum, for_html = for_html)
+        if for_html:
+            outstr += "</div>"
         return outstr
     
     def to_mse(self, print_raw = False, vdump = False):
@@ -794,7 +842,7 @@ class Card:
             mtext = transforms.text_unpass_2_counters(mtext)
             mtext = transforms.text_unpass_3_uncast(mtext)
             mtext = transforms.text_unpass_4_unary(mtext)
-            mtext = transforms.text_unpass_5_symbols(mtext, False)
+            mtext = transforms.text_unpass_5_symbols(mtext, False, False)
             mtext = sentencecase(mtext)
             # I don't really want these MSE specific passes in transforms,
             # but they could be pulled out separately somewhere else in here.
@@ -861,7 +909,7 @@ class Card:
                 mtext2 = transforms.text_unpass_2_counters(mtext2)
                 mtext2 = transforms.text_unpass_3_uncast(mtext2)
                 mtext2 = transforms.text_unpass_4_unary(mtext2)
-                mtext2 = transforms.text_unpass_5_symbols(mtext2, False)
+                mtext2 = transforms.text_unpass_5_symbols(mtext2, False, False)
                 mtext2 = sentencecase(mtext2)
                 mtext2 = mtext2.replace(utils.this_marker, '<atom-cardname><nospellcheck>'
                                       + utils.this_marker + '</nospellcheck></atom-cardname>')

lib/jdecode.py

@@ -1,6 +1,7 @@
 import json
 
-import config
+import utils
+import cardlib
 
 def mtg_open_json(fname, verbose = False):
 
@@ -16,7 +17,7 @@ def mtg_open_json(fname, verbose = False):
         setname = set['name']
         
         for card in set['cards']:
-            card[config.json_field_set_name] = setname
+            card[utils.json_field_set_name] = setname
 
             cardnumber = None
             if 'number' in card:
@@ -48,7 +49,7 @@ def mtg_open_json(fname, verbose = False):
         if aside_uid in asides:
             # the second check handles the brothers yamazaki edge case
             if not asides[aside_uid]['name'] == bsides[uid]['name']:
-                asides[aside_uid][config.json_field_bside] = bsides[uid]
+                asides[aside_uid][utils.json_field_bside] = bsides[uid]
         else:
             pass
             # this exposes some coldsnap theme deck bsides that aren't
@@ -59,3 +60,109 @@ def mtg_open_json(fname, verbose = False):
     if verbose:
         print 'Opened ' + str(len(allcards)) + ' uniquely named cards.'
     return allcards
+
+# filters to ignore some undesirable cards, only used when opening json
+def default_exclude_sets(cardset):
+    return cardset == 'Unglued' or cardset == 'Unhinged' or cardset == 'Celebration'
+
+def default_exclude_types(cardtype):
+    return cardtype in ['conspiracy']
+
+def default_exclude_layouts(layout):
+    return layout in ['token', 'plane', 'scheme', 'phenomenon', 'vanguard']
+
+# centralized logic for opening files of cards, either encoded or json
+def mtg_open_file(fname, verbose = False,
+                  linetrans = True, fmt_ordered = cardlib.fmt_ordered_default,
+                  exclude_sets = default_exclude_sets,
+                  exclude_types = default_exclude_types,
+                  exclude_layouts = default_exclude_layouts):
+
+    cards = []
+    valid = 0
+    skipped = 0
+    invalid = 0
+    unparsed = 0
+
+    if fname[-5:] == '.json':
+        if verbose:
+            print 'This looks like a json file: ' + fname
+        json_srcs = mtg_open_json(fname, verbose)
+        # sorted for stability
+        for json_cardname in sorted(json_srcs):
+            if len(json_srcs[json_cardname]) > 0:
+                jcards = json_srcs[json_cardname]
+
+                # look for a normal rarity version, in a set we can use
+                idx = 0
+                card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                while (idx < len(jcards)
+                       and (card.rarity == utils.rarity_special_marker
+                            or exclude_sets(jcards[idx][utils.json_field_set_name]))):
+                    idx += 1
+                    if idx < len(jcards):
+                        card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                # if there isn't one, settle with index 0
+                if idx >= len(jcards):
+                    idx = 0
+                    card = cardlib.Card(jcards[idx], linetrans=linetrans)
+                # we could go back and look for a card satisfying one of the criteria,
+                # but eh
+
+                skip = False
+                if (exclude_sets(jcards[idx][utils.json_field_set_name])
+                    or exclude_layouts(jcards[idx]['layout'])):
+                    skip = True
+                for cardtype in card.types:
+                    if exclude_types(cardtype):
+                        skip = True
+                if skip:
+                    skipped += 1
+                    continue
+
+                if card.valid:
+                    valid += 1
+                    cards += [card]
+                elif card.parsed:
+                    invalid += 1
+                else:
+                    unparsed += 1
+
+    # fall back to opening a normal encoded file
+    else:
+        if verbose:
+            print 'Opening encoded card file: ' + fname
+        with open(fname, 'rt') as f:
+            text = f.read()
+        for card_src in text.split(utils.cardsep):
+            if card_src:
+                card = cardlib.Card(card_src, fmt_ordered=fmt_ordered)
+                if card.valid:
+                    valid += 1
+                    cards += [card]
+                elif card.parsed:
+                    invalid += 1
+                else:
+                    unparsed += 1
+
+    if verbose:
+        print (str(valid) + ' valid, ' + str(skipped) + ' skipped, '
+               + str(invalid) + ' invalid, ' + str(unparsed) + ' failed to parse.')
+
+    good_count = 0
+    bad_count = 0
+    for card in cards:
+        if not card.parsed and not card.text.text:
+            bad_count += 1
+        elif len(card.name) > 50 or len(card.rarity) > 3:
+            bad_count += 1
+        else:
+            good_count += 1
+        if good_count + bad_count > 15:
+            break
+    # random heuristic
+    if bad_count > 10:
+        print 'WARNING: Saw a bunch of unparsed cards:'
+        print '         Is this a legacy format, you may need to specify the field order.'
+
+    return cards

lib/manalib.py

@@ -108,12 +108,13 @@ class Manacost:
         return utils.mana_untranslate(utils.mana_open_delimiter + ''.join(self.sequence)
                                       + utils.mana_close_delimiter)
 
-    def format(self, for_forum = False):
+    def format(self, for_forum = False, for_html = False):
         if self.none:
             return '_NOCOST_'
+        
         else:
             return utils.mana_untranslate(utils.mana_open_delimiter + ''.join(self.sequence)
-                                          + utils.mana_close_delimiter, for_forum)
+                                          + utils.mana_close_delimiter, for_forum, for_html)
 
     def encode(self, randomize = False):
         if self.none:
@@ -177,10 +178,10 @@ class Manatext:
             text = text.replace(utils.reserved_mana_marker, str(cost), 1)
         return text
 
-    def format(self, for_forum = False):
+    def format(self, for_forum = False, for_html = False):
         text = self.text
         for cost in self.costs:
-            text = text.replace(utils.reserved_mana_marker, cost.format(for_forum = for_forum), 1)
+            text = text.replace(utils.reserved_mana_marker, cost.format(for_forum = for_forum, for_html = for_html), 1)
         return text
 
     def encode(self, randomize = False):

lib/transforms.py

@@ -399,13 +399,55 @@ def text_pass_8_equip(s):
 
 
 def text_pass_9_newlines(s):
-    return s.replace('\n', '\\')
+    return s.replace('\n', utils.newline)
 
 
 def text_pass_10_symbols(s):
     return utils.to_symbols(s)
 
 
+# reorder the lines of text into a canonical form:
+# first enchant and equip
+# then other keywords, one per line (things with no period on the end)
+# then other abilities
+# then kicker and countertype last of all
+def text_pass_11_linetrans(s):
+    # let's just not deal with level up
+    if 'level up' in s:
+        return s
+
+    prelines = []
+    keylines = []
+    mainlines = []
+    postlines = []
+
+    lines = s.split(utils.newline)
+    for line in lines:
+        if not '.' in line:
+            # because this is inconsistent
+            line = line.replace(';', ',')
+            sublines = line.split(',')
+            for subline in sublines:
+                if 'equip' in subline or 'enchant' in subline:
+                    prelines += [subline.strip()]
+                elif 'countertype' in subline or 'kicker' in subline:
+                    postlines += [subline.strip()]
+                else:
+                    keylines += [subline.strip()]
+        elif u'\u2014' in line and not u' \u2014 ' in line:
+            if 'equip' in line or 'enchant' in line:
+                prelines += [line.strip()]
+            elif 'countertype' in line or 'kicker' in line:
+                postlines += [line.strip()]
+            else:
+                keylines += [line.strip()]
+        else:
+            mainlines += [line.strip()]
+
+    alllines = prelines + keylines + mainlines + postlines
+    return utils.newline.join(alllines)
+
+
 # Text unpasses, for decoding. All assume the text inside a Manatext, so don't do anything
 # weird with the mana cost symbol.
 
@@ -473,8 +515,8 @@ def text_unpass_4_unary(s):
     return utils.from_unary(s)
 
 
-def text_unpass_5_symbols(s, for_forum):
-    return utils.from_symbols(s, for_forum = for_forum)
+def text_unpass_5_symbols(s, for_forum, for_html):
+    return utils.from_symbols(s, for_forum = for_forum, for_html = for_html)
 
 
 def text_unpass_6_cardname(s, name):