hamcha/mtgencode
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' of github.com:billzorn/mtgencode

Browse source
This commit is contained in:
Bill Zorn 2015-12-03 15:21:24 -08:00
commit 28c3609c37
1 changed files with 26 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
scripts/collect_checkpoints.py
View file

@ -3,13 +3,20 @@ import sys
import os import os
import shutil import shutil
def cleanup_dump(dumpstr):
cardfrags = dumpstr.split('\n\n')
if len(cardfrags) < 4:
return ''
else:
return '\n\n'.join(cardfrags[2:-1]) + '\n\n'
def identify_checkpoints(basedir, ident): def identify_checkpoints(basedir, ident):
cp_infos = [] cp_infos = []
for path in os.listdir(cpdir): for path in os.listdir(basedir):
fullpath = os.path.join(basedir, path) fullpath = os.path.join(basedir, path)
if not os.path.isfile(fullpath): if not os.path.isfile(fullpath):
continue continue
if not (name[:13] == 'lm_lstm_epoch' and name[-4:] == '.txt'): if not (path[:13] == 'lm_lstm_epoch' and path[-4:] == '.txt'):
continue continue
if not ident in path: if not ident in path:
continue continue
@ -33,30 +40,36 @@ def identify_checkpoints(basedir, ident):
return cp_infos return cp_infos
def process_dir(basedir, targetdir, ident, copy_cp = False, verbose = False): def process_dir(basedir, targetdir, ident, copy_cp = False, verbose = False):
(basepath, basedirname) = os.path.split(basedir)
if basedirname == '':
(basepath, basedirname) = os.path.split(basepath)
cp_infos = identify_checkpoints(basedir, ident) cp_infos = identify_checkpoints(basedir, ident)
for (dpath, cpath, (epoch, vloss, temp)) in cp_infos: for (dpath, cpath, (epoch, vloss, temp)) in cp_infos:
if verbose: if verbose:
print('found dumpfile ' + dpath) print('found dumpfile ' + dpath)
dname = basedir + '_epoch' + epoch + '_' + vloss + '.' + ident + '.' + temp + '.txt' dname = basedirname + '_epoch' + epoch + '_' + vloss + '.' + ident + '.' + temp + '.txt'
cname = basedir + '_epoch' + epoch + '_' + vloss + '.t7' cname = basedirname + '_epoch' + epoch + '_' + vloss + '.t7'
tdpath = os.path.join(targetdir, dname) tdpath = os.path.join(targetdir, dname)
tcpath = os.path.join(targetdir, cname) tcpath = os.path.join(targetdir, cname)
if verbose: if verbose:
print('cp ' + dpath + ' ' + tdpath) print(' cpx ' + dpath + ' ' + tdpath)
#shutil.copy(dpath, tdpath) with open(dpath, 'rt') as infile:
with open(tdpath, 'wt') as outfile:
outfile.write(cleanup_dump(infile.read()))
if copy_cp: if copy_cp:
if os.path.isfile('cpath'): if os.path.isfile(cpath):
if verbose: if verbose:
print(' cp ' + cpath + ' ' + tcpath) print(' cp ' + cpath + ' ' + tcpath)
#shutil.copy(cpath, tcpath) shutil.copy(cpath, tcpath)
if copy_cp and len(cp_infos) > 0: if copy_cp and len(cp_infos) > 0:
cmdpath = os.path.join(basedir, 'command.txt') cmdpath = os.path.join(basedir, 'command.txt')
tcmdpath = os.path.join(targetdir, basedir + '.command') tcmdpath = os.path.join(targetdir, basedirname + '.command')
if os.path.isfile('cpath'): if os.path.isfile(cmdpath):
if verbose: if verbose:
print(' cp ' + cmdpath + ' ' + tcmdpath) print(' cp ' + cmdpath + ' ' + tcmdpath)
#shutil.copy(cmdpath, tcmdpath) shutil.copy(cmdpath, tcmdpath)
for path in os.listdir(basedir): for path in os.listdir(basedir):
fullpath = os.path.join(basedir, path) fullpath = os.path.join(basedir, path)