added line transforms, not totally sure if they work yet

encode.py

@@ -84,17 +84,17 @@ def main(fname, oname = None, verbose = True, encoding = 'std',
 
                 # look for a normal rarity version, in a set we can use
                 idx = 0
-                card = cardlib.Card(jcards[idx])
+                card = cardlib.Card(jcards[idx], linetrans = line_transformations)
                 while (idx < len(jcards)
                        and (card.rarity == utils.rarity_special_marker 
                             or exclude_sets(jcards[idx][utils.json_field_set_name]))):
                     idx += 1
                     if idx < len(jcards):
-                        card = cardlib.Card(jcards[idx])
+                        card = cardlib.Card(jcards[idx], linetrans = line_transformations)
                 # if there isn't one, settle with index 0
                 if idx >= len(jcards):
                     idx = 0
-                    card = cardlib.Card(jcards[idx])
+                    card = cardlib.Card(jcards[idx], linetrans = line_transformations)
                 # we could go back and look for a card satisfying one of the criteria,
                 # but eh
 

lib/cardlib.py

@@ -223,7 +223,7 @@ def fields_check_valid(fields):
 # releaseDate - string
 # starter - boolean
 
-def fields_from_json(src_json):
+def fields_from_json(src_json, linetrans = True):
     parsed = True
     valid = True
     fields = {}
@@ -301,6 +301,8 @@ def fields_from_json(src_json):
         text_val = transforms.text_pass_8_equip(text_val)
         text_val = transforms.text_pass_9_newlines(text_val)
         text_val = transforms.text_pass_10_symbols(text_val)
+        if linetrans:
+            text_val = transforms.text_pass_11_linetrans(text_val)
         text_val = utils.to_ascii(text_val)
         text_val = text_val.strip()
         mtext = Manatext(text_val, fmt = 'json')
@@ -389,7 +391,7 @@ class Card:
 
     def __init__(self, src, fmt_ordered = fmt_ordered_default, 
                             fmt_labeled = fmt_labeled_default, 
-                            fieldsep = utils.fieldsep):
+                            fieldsep = utils.fieldsep, linetrans = True):
         # source fields, exactly one will be set
         self.json = None
         self.raw = None
@@ -425,8 +427,9 @@ class Card:
                 self.bside = Card(src[utils.json_field_bside],
                                   fmt_ordered = fmt_ordered,
                                   fmt_labeled = fmt_labeled,
-                                  fieldsep = fieldsep)
-            p_success, v_success, parsed_fields = fields_from_json(src)
+                                  fieldsep = fieldsep,
+                                  linetrans = linetrans)
+            p_success, v_success, parsed_fields = fields_from_json(src, linetrans = linetrans)
             self.parsed = p_success
             self.valid = v_success
             self.fields = parsed_fields
@@ -438,7 +441,8 @@ class Card:
                 self.bside = Card(utils.bsidesep.join(sides[1:]), 
                                   fmt_ordered = fmt_ordered,
                                   fmt_labeled = fmt_labeled,
-                                  fieldsep = fieldsep)
+                                  fieldsep = fieldsep,
+                                  linetrans = linetrans)
             p_success, v_success, parsed_fields = fields_from_format(sides[0], fmt_ordered, 
                                                                      fmt_labeled,  fieldsep)
             self.parsed = p_success

lib/transforms.py

@@ -399,13 +399,56 @@ def text_pass_8_equip(s):
 
 
 def text_pass_9_newlines(s):
-    return s.replace('\n', '\\')
+    return s.replace('\n', utils.newline)
 
 
 def text_pass_10_symbols(s):
     return utils.to_symbols(s)
 
 
+# reorder the lines of text into a canonical form:
+# first enchant and equip
+# then other keywords, one per line (things with no period on the end)
+# then other abilities
+# then kicker and countertype last of all
+def text_pass_11_linetrans(s):
+    # let's just not deal with level up
+    if 'level up' in s:
+        return s
+
+    prelines = []
+    keylines = []
+    mainlines = []
+    postlines = []
+
+    lines = s.split(utils.newline)
+    for line in lines:
+        if not '.' in line:
+            # because this is inconsistent
+            line = line.replace(';', ',')
+            sublines = line.split(',')
+            for subline in sublines:
+                if 'equip' in subline or 'enchant' in subline:
+                    prelines += [subline]
+                elif 'countertype' or 'kicker' in subline:
+                    postlines += [subline]
+                else:
+                    keylines += [subline]
+        elif u'\u2014' in line and not u' \u2014 ' in line:
+            if 'equip' in line or 'enchant' in line:
+                prelines += [line]
+            elif 'countertype' or 'kicker' in line:
+                postlines += [line]
+            else:
+                keylines += [line]
+        else:
+            mainlines += [line]
+            print line.encode('utf-8')
+
+    alllines = prelines + keylines + mainlines + postlines
+    return utils.newline.join(alllines)
+
+
 # Text unpasses, for decoding. All assume the text inside a Manatext, so don't do anything
 # weird with the mana cost symbol.