1 """@package forcebalance.nifty Nifty functions, intended to be imported by any module within ForceBalance.
5 - Math: Variable manipulation, linear algebra, least squares polynomial fitting
6 - Pickle: Expand Python's own pickle to accommodate writing XML etree objects
7 - Commands for submitting things to the Work Queue
8 - Various file and process management functions
9 - Development stuff (not commonly used)
11 Named after the mighty Sniffy Handy Nifty (King Sniffy)
17 from select
import select
18 import os, sys, shutil
19 from re
import match, sub
22 from numpy
import array, diag, dot, eye, mat, mean, transpose
23 from numpy.linalg
import norm, svd
28 from subprocess
import PIPE, STDOUT
29 from collections
import OrderedDict, defaultdict
33 logger = getLogger(__name__)
49 def pvec1d(vec1d, precision=1, loglevel=INFO):
50 """Printout of a 1-D vector.
52 @param[in] vec1d a 1-D vector
55 for i
in range(v2a.shape[0]):
56 logger.log(loglevel,
"%% .%ie " % precision % v2a[i])
57 logger.log(loglevel,
'\n')
59 def pmat2d(mat2d, precision=1, loglevel=INFO):
60 """Printout of a 2-D matrix.
62 @param[in] mat2d a 2-D matrix
65 for i
in range(m2a.shape[0]):
66 for j
in range(m2a.shape[1]):
67 logger.log(loglevel,
"%% .%ie " % precision % m2a[i][j])
68 logger.log(loglevel,
'\n')
71 return [[len(list(group)),name]
for name, group
in itertools.groupby(l)]
75 begins = np.array([sum([k[0]
for k
in e][:j])
for j,i
in enumerate(e)
if i[1] == 1])
76 lens = np.array([i[0]
for i
in e
if i[1] == 1])
77 return [(i, i+j)
for i, j
in zip(begins, lens)]
86 LL = [i
in L
for i
in range(L[-1])]
87 return ','.join(
'%i-%i' % (i[0]+1,i[1])
if (i[1]-1 > i[0])
else '%i' % (i[0]+1)
for i
in segments(
encode(LL)))
94 for w
in s.split(
','):
102 logger.warning(
"Dash-separated list cannot exceed length 2\n")
104 if a < 0
or b <= 0
or b <= a:
106 logger.warning(
"Items in list cannot be zero or negative: %d %d\n" % (a, b))
108 logger.warning(
"Second number cannot be larger than first: %d %d\n" % (a, b))
111 if any([i
in L
for i
in newL]):
112 logger.warning(
"Duplicate entries found in list\n")
116 logger.warning(
"List is out of order\n")
119 raise Exception(
'Invalid string for converting to list of numbers: %s' % s)
124 def printcool(text,sym="#",bold=False,color=2,ansi=None,bottom='-',minwidth=50,center=True):
125 """Cool-looking printout for slick formatting of output.
127 @param[in] text The string that the printout is based upon. This function
128 will print out the string, ANSI-colored and enclosed in the symbol
130 <tt> ################# </tt>\n
131 <tt> ### I am cool ### </tt>\n
132 <tt> ################# </tt>
133 @param[in] sym The surrounding symbol\n
134 @param[in] bold Whether to use bold print
136 @param[in] color The ANSI color:\n
145 @param[in] bottom The symbol for the bottom bar
147 @param[in] minwidth The minimum width for the box, if the text is very short
148 then we insert the appropriate number of padding spaces
150 @return bar The bottom bar is returned for the user to print later, e.g. to mark off a 'section'
153 return len(sub(
"\x1b\[[0-9;]*m",
"",line))
154 text = text.split(
'\n')
155 width = max(minwidth,max([newlen(line)
for line
in text]))
156 bar =
''.join([
"=" for i
in range(width + 6)])
157 bar = sym + bar + sym
159 logger.info(
'\r'+bar +
'\n')
162 padleft =
' ' * ((width - newlen(line)) / 2)
165 padright =
' '* (width - newlen(line) - len(padleft))
168 logger.info(
"%s| \x1b[%sm%s " % (sym, ansi, padleft)+line+
" %s\x1b[0m |%s\n" % (padright, sym))
170 if color == 0
and bold:
171 logger.info(
"%s| \x1b[1m%s " % (sym, padleft) + line +
" %s\x1b[0m |%s\n" % (padright, sym))
173 logger.info(
"%s| %s " % (sym, padleft)+line+
" %s |%s\n" % (padright, sym))
175 logger.info(
"%s| \x1b[%s9%im%s " % (sym, bold
and "1;" or "", color, padleft)+line+
" %s\x1b[0m |%s\n" % (padright, sym))
182 logger.info(bar +
'\n')
183 botbar =
''.join([bottom
for i
in range(width + 8)])
186 def printcool_dictionary(Dict,title="General options",bold=False,color=2,keywidth=25,topwidth=50,center=True,leftpad=0):
187 """See documentation for printcool; this is a nice way to print out keys/values in a dictionary.
189 The keys in the dictionary are sorted before printing out.
191 @param[in] dict The dictionary to be printed
192 @param[in] title The title of the printout
194 if Dict ==
None:
return
195 bar =
printcool(title,bold=bold,color=color,minwidth=topwidth,center=center)
196 def magic_string(str):
200 return eval(
"\'%%-%is\' %% '%s'" % (keywidth,str.replace(
"'",
"\\'").replace(
'"',
'\\"')))
201 if isinstance(Dict, OrderedDict):
202 logger.info(
'\n'.join([
' '*leftpad +
"%s %s " % (magic_string(str(key)),str(Dict[key]))
for key
in Dict
if Dict[key] !=
None]))
204 logger.info(
'\n'.join([
' '*leftpad +
"%s %s " % (magic_string(str(key)),str(Dict[key]))
for key
in sorted([i
for i
in Dict])
if Dict[key] !=
None]))
205 logger.info(
"\n%s" % bar)
211 """ONLY matches integers! If you have a decimal point? None shall pass!
213 @param[in] word String (for instance, '123', '153.0', '2.', '-354')
214 @return answer Boolean which specifies whether the string is an integer (only +/- sign followed by digits)
217 return match(
'^[-+]?[0-9]+$',word)
220 """Matches ANY number; it can be a decimal, scientific notation, what have you
221 CAUTION - this will also match an integer.
223 @param[in] word String (for instance, '123', '153.0', '2.', '-354')
224 @return answer Boolean which specifies whether the string is any number
227 return match(
'^[-+]?[0-9]*\.?[0-9]*([eEdD][-+]?[0-9]+)?$',word)
230 """Matches things with a decimal only; see isint and isfloat.
232 @param[in] word String (for instance, '123', '153.0', '2.', '-354')
233 @return answer Boolean which specifies whether the string is a number with a decimal point
239 """Returns a big number if we encounter NaN.
241 @param[in] word The string to be converted
242 @return answer The string converted to a float; if not a float, return 1e10
243 @todo I could use suggestions for making this better.
249 logger.info(
"Setting %s to % .1e\n" % big)
254 Given any list, array, or matrix, return a 1-column matrix.
257 vec = The input vector that is to be made into a column
262 return mat(array(vec).reshape(-1, 1))
265 """Given any list, array, or matrix, return a 1-row matrix.
267 @param[in] vec The input vector that is to be made into a row
269 @return answer A row matrix
271 return mat(array(vec).reshape(1, -1))
274 """Given any list, array, or matrix, return a single-index array.
276 @param[in] vec The data to be flattened
277 @return answer The flattened data
279 return array(vec).reshape(-1)
285 """Given two vectors vec1 and vec2, project out the component of vec1
286 that is along the vec2-direction.
288 @param[in] vec1 The projectee (i.e. output is some modified version of vec1)
289 @param[in] vec2 The projector (component subtracted out from vec1 is parallel to this)
290 @return answer A copy of vec1 but with the vec2-component projected out.
292 v2u = vec2/norm(vec2)
293 return vec1 - v2u*dot(vec1, v2u)
299 Invert a matrix using singular value decomposition.
300 @param[in] X The matrix to be inverted
301 @param[in] thresh The SVD threshold; eigenvalues below this are not inverted but set to zero
302 @return Xt The inverted matrix
306 u,s,vh = svd(X, full_matrices=0)
307 uh = mat(transpose(u))
308 v = mat(transpose(vh))
310 for i
in range(s.shape[0]):
311 if abs(s[i]) > thresh:
327 | 1 (x0) (x0)^2 (x0)^3 |
328 | 1 (x1) (x1)^2 (x1)^3 |
329 | 1 (x2) (x2)^2 (x2)^3 |
330 | 1 (x3) (x3)^2 (x3)^3 |
331 | 1 (x4) (x4)^2 (x4)^3 |
336 @param[in] X (2-D array) An array of X-values (see above)
337 @param[in] Y (array) An array of Y-values (only used in getting the least squares coefficients)
338 @param[in] w (array) An array of weights, hopefully normalized to one.
339 @param[out] Beta The least-squares coefficients
340 @param[out] Hat The hat matrix that takes linear combinations of data y-values to give fitted y-values (weights)
341 @param[out] yfit The fitted y-values
342 @param[out] MPPI The Moore-Penrose pseudoinverse (multiply by Y to get least-squares coefficients, multiply by dY/dk to get derivatives of least-squares coefficients)
350 logger.warning(
"Argh? It seems like this problem is underdetermined!\n")
354 warn_press_key(
"The weight array length (%i) must be the same as the number of 'X' data points (%i)!" % len(w), n_x)
356 WH = mat(diag(w**0.5))
364 MPPI = np.linalg.pinv(WH*X)
370 return Beta, Hat, yfit, MPPI
378 Compute the (cross) statistical inefficiency of (two) timeseries.
381 The same timeseries can be used for both A_n and B_n to get the autocorrelation statistical inefficiency.
382 The fast method described in Ref [1] is used to compute g.
385 [1] J. D. Chodera, W. C. Swope, J. W. Pitera, C. Seok, and K. A. Dill. Use of the weighted
386 histogram analysis method for the analysis of simulated and parallel tempering simulations.
387 JCTC 3(1):26-41, 2007.
391 Compute statistical inefficiency of timeseries data with known correlation time.
393 >>> import timeseries
394 >>> A_n = timeseries.generateCorrelatedTimeseries(N=100000, tau=5.0)
395 >>> g = statisticalInefficiency(A_n, fast=True)
397 @param[in] A_n (required, numpy array) - A_n[n] is nth value of
398 timeseries A. Length is deduced from vector.
400 @param[in] B_n (optional, numpy array) - B_n[n] is nth value of
401 timeseries B. Length is deduced from vector. If supplied, the
402 cross-correlation of timeseries A and B will be estimated instead of
403 the autocorrelation of timeseries A.
405 @param[in] fast (optional, boolean) - if True, will use faster (but
406 less accurate) method to estimate correlation time, described in
407 Ref. [1] (default: False)
409 @param[in] mintime (optional, int) - minimum amount of correlation
410 function to compute (default: 3) The algorithm terminates after
411 computing the correlation time out to mintime when the correlation
412 function furst goes negative. Note that this time may need to be
413 increased if there is a strong initial negative peak in the
414 correlation function.
416 @return g The estimated statistical inefficiency (equal to 1 + 2
417 tau, where tau is the correlation time). We enforce g >= 1.0.
430 if(A_n.shape != B_n.shape):
431 raise ParameterError(
'A_n and B_n must have same dimensions.')
438 dA_n = A_n.astype(np.float64) - mu_A
439 dB_n = B_n.astype(np.float64) - mu_B
441 sigma2_AB = (dA_n * dB_n).mean()
445 logger.warning(
'Sample covariance sigma_AB^2 = 0 -- cannot compute statistical inefficiency')
455 C = sum( dA_n[0:(N-t)]*dB_n[t:N] + dB_n[0:(N-t)]*dA_n[t:N] ) / (2.0 * float(N-t) * sigma2_AB)
458 if (C <= 0.0)
and (t > mintime):
461 g += 2.0 * C * (1.0 - float(t)/float(N)) * float(increment)
465 if fast: increment += 1
467 if (g < 1.0): g = 1.0
475 from lxml
import etree
477 logger.warning(
"lxml module import failed (You can't use OpenMM or XML force fields)\n")
483 """ A subclass of the python Pickler that implements pickling of _ElementTree types. """
484 def __init__(self, file, protocol=None):
485 pickle.Pickler.__init__(self, file, protocol)
487 def save_etree(self, obj):
490 String = etree.tostring(obj)
493 logger.error(
"self.bin is True, not sure what to do with myself\n")
496 self.write(XMLFILE + repr(String) +
'\n')
499 warn_once(
"Cannot save XML files; if using OpenMM install libxml2+libxslt+lxml. Otherwise don't worry.")
501 self.dispatch[etree._ElementTree] = save_etree
503 warn_once(
"Cannot save XML files; if using OpenMM install libxml2+libxslt+lxml. Otherwise don't worry.")
506 """ A subclass of the python Unpickler that implements unpickling of _ElementTree types. """
508 pickle.Unpickler.__init__(self, file)
509 def load_etree(self):
512 rep = self.readline()[:-1]
514 if rep.startswith(q):
515 if not rep.endswith(q):
516 raise ValueError,
"insecure string pickle"
517 rep = rep[len(q):-len(q)]
520 raise ValueError,
"insecure string pickle"
522 self.append(etree.ElementTree(etree.fromstring(rep.decode(
"string-escape"))))
524 warn_once(
"Cannot load XML files; if using OpenMM install libxml2+libxslt+lxml. Otherwise don't worry.")
526 self.dispatch[XMLFILE] = load_etree
528 warn_once(
"Cannot load XML files; if using OpenMM install libxml2+libxslt+lxml. Otherwise don't worry.")
530 def lp_dump(obj, file, protocol=None):
531 """ Use this instead of pickle.dump for pickling anything that contains _ElementTree types. """
535 """ Use this instead of pickle.load for unpickling anything that contains _ElementTree types. """
544 logger.warning(
"Work Queue library import fail (You can't queue up jobs using Work Queue)\n")
550 WQIDS = defaultdict(list)
563 work_queue.set_debug_flag(
'all')
564 WORK_QUEUE = work_queue.WorkQueue(port=wq_port, catalog=
True, exclusive=
False, shutdown=
False)
565 WORK_QUEUE.tasks_failed = 0
566 WORK_QUEUE.specify_name(
'forcebalance')
568 WORK_QUEUE.specify_keepalive_interval(8640000)
572 global WORK_QUEUE, WQIDS
574 WQIDS = defaultdict(list)
576 def queue_up(wq, command, input_files, output_files, tgt=None, verbose=True):
578 Submit a job to the Work Queue.
580 @param[in] wq (Work Queue Object)
581 @param[in] command (string) The command to run on the remote worker.
582 @param[in] input_files (list of files) A list of locations of the input files.
583 @param[in] output_files (list of files) A list of locations of the output files.
586 task = work_queue.Task(command)
588 for f
in input_files:
589 lf = os.path.join(cwd,f)
590 task.specify_input_file(lf,f,cache=
False)
591 for f
in output_files:
592 lf = os.path.join(cwd,f)
593 task.specify_output_file(lf,f,cache=
False)
594 task.specify_algorithm(work_queue.WORK_QUEUE_SCHEDULE_FCFS)
595 task.specify_tag(command)
596 taskid = wq.submit(task)
598 logger.info(
"Submitting command '%s' to the Work Queue, taskid %i\n" % (command, taskid))
600 WQIDS[tgt.name].append(taskid)
602 WQIDS[
"None"].append(taskid)
604 def queue_up_src_dest(wq, command, input_files, output_files, tgt=None, verbose=True):
606 Submit a job to the Work Queue. This function is a bit fancier in that we can explicitly
607 specify where the input files come from, and where the output files go to.
609 @param[in] wq (Work Queue Object)
610 @param[in] command (string) The command to run on the remote worker.
611 @param[in] input_files (list of 2-tuples) A list of local and
612 remote locations of the input files.
613 @param[in] output_files (list of 2-tuples) A list of local and
614 remote locations of the output files.
617 task = work_queue.Task(command)
618 for f
in input_files:
620 task.specify_input_file(f[0],f[1],cache=
False)
621 for f
in output_files:
623 task.specify_output_file(f[0],f[1],cache=
False)
624 task.specify_algorithm(work_queue.WORK_QUEUE_SCHEDULE_FCFS)
625 task.specify_tag(command)
626 taskid = wq.submit(task)
628 logger.info(
"Submitting command '%s' to the Work Queue, taskid %i\n" % (command, taskid))
630 WQIDS[tgt.name].append(taskid)
632 WQIDS[
"None"].append(taskid)
634 def wq_wait1(wq, wait_time=10, wait_intvl=1, print_time=60, verbose=False):
635 """ This function waits ten seconds to see if a task in the Work Queue has finished. """
637 if verbose: logger.info(
'---\n')
638 if wait_intvl >= wait_time:
639 wait_time = wait_intvl
642 numwaits = wait_time / wait_intvl
643 for sec
in range(numwaits):
644 task = wq.wait(wait_intvl)
646 exectime = task.cmd_execution_time/1000000
648 logger.info(
'A job has finished!\n')
649 logger.info(
'Job name = ' + task.tag +
'command = ' + task.command +
'\n')
650 logger.info(
"status = " + task.status +
'\n')
651 logger.info(
"return_status = " + task.return_status)
652 logger.info(
"result = " + task.result)
653 logger.info(
"host = " + task.hostname +
'\n')
654 logger.info(
"execution time = " + exectime)
655 logger.info(
"total_bytes_transferred = " + task.total_bytes_transferred +
'\n')
658 oldhost = task.hostname
661 if task.id
in WQIDS[tnm]:
663 WQIDS[tnm].remove(task.id)
664 taskid = wq.submit(task)
665 logger.warning(
"Command '%s' (task %i) failed on host %s (%i seconds), resubmitted: taskid %i\n" % (task.command, oldid, oldhost, exectime, taskid))
666 WQIDS[tgtname].append(taskid)
669 if exectime > print_time:
670 logger.info(
"Command '%s' (task %i) finished successfully on host %s (%i seconds)\n" % (task.command, task.id, task.hostname, exectime))
672 if task.id
in WQIDS[tnm]:
673 WQIDS[tnm].remove(task.id)
677 nbusy = wq.stats.workers_busy + wq.stats.workers_full
679 nbusy = wq.stats.workers_busy
682 Complete = wq.stats.total_tasks_complete - wq.tasks_failed
683 Total = wq.stats.total_tasks_dispatched - wq.tasks_failed
685 logger.warning(
"wq object has no tasks_failed attribute, please use createWorkQueue() function.\n")
686 Complete = wq.stats.total_tasks_complete
687 Total = wq.stats.total_tasks_dispatched
690 logger.info(
"Workers: %i init, %i ready, %i busy, %i total joined, %i total removed\n" \
691 % (wq.stats.workers_init, wq.stats.workers_ready, nbusy, wq.stats.total_workers_joined, wq.stats.total_workers_removed))
692 logger.info(
"Tasks: %i running, %i waiting, %i total dispatched, %i total complete\n" \
693 % (wq.stats.tasks_running,wq.stats.tasks_waiting,Total,Complete))
694 logger.info(
"Data: %i / %i kb sent/received\n" % (wq.stats.total_bytes_sent/1000, wq.stats.total_bytes_received/1024))
696 logger.info(
"\r%s : %i/%i workers busy; %i/%i jobs complete\r" %\
698 nbusy, (wq.stats.total_workers_joined - wq.stats.total_workers_removed),
700 if time.time() - wq_wait1.t0 > 900:
701 wq_wait1.t0 = time.time()
703 wq_wait1.t0 = time.time()
705 def wq_wait(wq, wait_time=10, wait_intvl=10, print_time=60, verbose=False):
706 """ This function waits until the work queue is completely empty. """
707 while not wq.empty():
708 wq_wait1(wq, wait_time=wait_time, wait_intvl=wait_intvl, print_time=print_time, verbose=verbose)
714 if os.path.exists(Dir):
715 if os.path.isdir(Dir):
pass
716 else:
raise Exception(
"Tried to create directory %s, it exists but isn't a directory" % newdir)
723 s = os.path.split(os.path.normpath(Dir))
724 if s[1] ==
'' or s[1] ==
'.' :
return []
728 if os.path.split(os.getcwd())[1] != Dir:
729 raise Exception(
"Trying to leave directory %s, but we're actually in directory %s (check your code)" % (Dir,os.path.split(os.getcwd())[1]))
734 specific_lst = [([
'mdrun',
'grompp',
'trjconv',
'g_energy',
'g_traj'],
"Make sure to install GROMACS and add it to your path (or set the gmxpath option)"),
735 ([
'force.mdin',
'stage.leap'],
"This file is needed for setting up AMBER force matching targets"),
736 ([
'conf.pdb',
'mono.pdb'],
"This file is needed for setting up OpenMM condensed phase property targets"),
737 ([
'liquid.xyz',
'liquid.key',
'mono.xyz',
'mono.key'],
"This file is needed for setting up OpenMM condensed phase property targets"),
738 ([
'dynamic',
'analyze',
'minimize',
'testgrad',
'vibrate',
'optimize',
'polarize',
'superpose'],
"Make sure to install TINKER and add it to your path (or set the tinkerpath option)"),
739 ([
'runcuda.sh',
'npt.py',
'npt_tinker.py'],
"This file belongs in the ForceBalance source directory, not sure why it is missing"),
740 ([
'input.xyz'],
"This file is needed for TINKER molecular property targets"),
741 ([
'.*key$',
'.*xyz$'],
"I am guessing this file is probably needed by TINKER"),
742 ([
'.*gro$',
'.*top$',
'.*itp$',
'.*mdp$',
'.*ndx$'],
"I am guessing this file is probably needed by GROMACS")
746 specific_dct = dict(list(itertools.chain(*[[(j,i[1])
for j
in i[0]]
for i
in specific_lst])))
749 fnm = os.path.split(fnm)[1]
751 for key
in specific_dct:
755 answer +=
"%s\n" % specific_dct[key]
759 if os.path.exists(src):
760 if os.path.exists(dest):
761 if os.path.islink(dest):
pass
762 else:
raise Exception(
"Tried to create symbolic link %s to %s, destination exists but isn't a symbolic link" % (src, dest))
764 os.symlink(src, dest)
766 raise Exception(
"Tried to create symbolic link %s to %s, but source file doesn't exist%s" % (src,dest,
MissingFileInspection(src)))
770 if os.path.exists(src):
771 if os.path.exists(dest):
772 if os.path.islink(dest):
773 raise Exception(
"Tried to copy %s to %s, destination exists but it's a symbolic link" % (src, dest))
775 shutil.copy2(src, dest)
777 raise Exception(
"Tried to copy %s to %s, but source file doesn't exist%s" % (src,dest,
MissingFileInspection(src)))
780 for fnm
in os.listdir(abssrcdir):
781 srcfnm = os.path.join(abssrcdir, fnm)
782 destfnm = os.path.join(absdestdir, fnm)
783 if os.path.isfile(srcfnm):
784 if not os.path.exists(destfnm):
786 os.symlink(srcfnm, destfnm)
789 """ Remove the file if it exists (doesn't return an error). """
790 if os.path.exists(fnm):
796 return os.path.split(os.popen(
'which %s 2> /dev/null' % fnm).readlines()[0].strip())[0]
800 def _exec(command, print_to_screen = False, outfnm = None, logfnm = None, stdin = "", print_command = True, copy_stderr = False, persist = False, expand_cr=False, **kwargs):
801 """Runs command line using subprocess, optionally returning stdout.
803 command (required) = Name of the command you want to execute
804 outfnm (optional) = Name of the output file name (overwritten if exists)
805 logfnm (optional) = Name of the log file name (appended if exists)
806 stdin (optional) = A string to be passed to stdin, as if it were typed (use newline character to mimic Enter key)
807 print_command = Whether to print the command.
808 copy_stderr = Whether to copy the stderr stream to the stdout stream; useful for GROMACS which prints out everything to stderr (argh.)
809 expand_cr = Whether to expand carriage returns into newlines (useful for GROMACS mdrun).
810 persist = Continue execution even if the command gives a nonzero return code.
813 cmd_options={
'shell':(type(command)
is str),
'stdin':PIPE,
'stdout':PIPE,
'stderr':PIPE,
'universal_newlines':expand_cr}
818 with open(logfnm,
'a+')
as f:
822 with open(outfnm,
'w+' if wtf.first
else 'a+')
as f:
829 if stdin ==
None: stdin =
""
832 logger.info(
"Executing process: \x1b[92m%-50s\x1b[0m%s%s%s\n" % (
' '.join(command)
if type(command)
is list
else command,
833 " Output: %s" % outfnm
if outfnm !=
None else "",
834 " Append: %s" % logfnm
if logfnm !=
None else "",
835 (
" Stdin: %s" % stdin.replace(
'\n',
'\\n'))
if stdin
else ""))
836 wtf(
"Executing process: %s%s\n" % (command, (
" Stdin: %s" % stdin.replace(
'\n',
'\\n'))
if stdin
else ""))
838 cmd_options.update(kwargs)
839 p = subprocess.Popen(command, **cmd_options)
848 reads = [p.stdout.fileno(), p.stderr.fileno()]
849 ret = select(reads, [], [])
851 if fd == p.stdout.fileno():
852 read = p.stdout.readline()
853 if print_to_screen: sys.stdout.write(read)
856 if fd == p.stderr.fileno():
857 read = p.stderr.readline()
858 if print_to_screen: sys.stderr.write(read)
868 if p.returncode != 0:
870 logger.warning(
"Received an error message:\n")
871 logger.warning(
"\n[====] \x1b[91mError Message\x1b[0m [====]\n")
872 logger.warning(stderr)
873 logger.warning(
"[====] \x1b[91mEnd o'Message\x1b[0m [====]\n")
875 logger.info(
"%s gave a return code of %i (it may have crashed) -- carrying on\n" % (command, p.returncode))
880 raise Exception(
"\x1b[1;94m%s\x1b[0m gave a return code of %i (\x1b[91mit may have crashed\x1b[0m)\n" % (command, p.returncode))
883 return stdout.split(
'\n')
886 if type(warning)
is str:
887 logger.warning(warning +
'\n')
888 elif type(warning)
is list:
890 logger.warning(line +
'\n')
892 logger.warning(
"You're not supposed to pass me a variable of this type: " + str(type(warning)))
893 if sys.stdin.isatty():
894 logger.warning(
"\x1b[1;91mPress Enter or wait %i seconds (I assume no responsibility for what happens after this!)\x1b[0m\n" % timeout)
895 try: rlist, wlist, xlist = select([sys.stdin], [], [], timeout)
899 """ Prints a warning but will only do so once in a given run. """
902 if warnhash
in warn_once.already:
904 warn_once.already.add(warnhash)
905 if type(warning)
is str:
906 logger.info(warning +
'\n')
907 elif type(warning)
is list:
909 logger.info(line +
'\n')
910 warn_once.already = set()
917 Similar to the bultin function map(). But spawn a thread for each argument
918 and apply `func` concurrently.
920 Note: unlike map(), we cannot take an iterable argument. `data` should be an
929 result[i] = func(data[i])
931 threads = [threading.Thread(target=task_wrapper, args=(i,))
for i
in xrange(N)]
942 This function be given any of several variable types
943 (single file name, file object, or list of lines, or a list of the above)
944 and give a list of files:
946 [file1, file2, file3 ... ]
948 each of which can then be iterated over:
950 [[file1_line1, file1_line2 ... ], [file2_line1, file2_line2 ... ]]
955 elif type(arg) == file:
958 elif type(arg) == list:
959 if all([type(l) == str
for l
in arg]):
961 if any([match(
"^.*\n$",l)
for l
in arg]):
965 fins = [open(l)
for l
in arg]
966 elif all([type(l) == file
or type(l) == list
for l
in arg]):
969 logger.info(
"What did you give this program as input?\n")
970 logger.info(str(arg) +
'\n')
973 logger.info(
"What did you give this program as input?\n")
974 logger.info(str(arg) +
'\n')
A subclass of the python Pickler that implements pickling of _ElementTree types.
def multiopen
This function be given any of several variable types (single file name, file object, or list of lines, or a list of the above) and give a list of files:
def pmat2d
Printout of a 2-D matrix.
def orthogonalize
Given two vectors vec1 and vec2, project out the component of vec1 that is along the vec2-direction...
def printcool_dictionary
See documentation for printcool; this is a nice way to print out keys/values in a dictionary...
def queue_up
Submit a job to the Work Queue.
def remove_if_exists
Remove the file if it exists (doesn't return an error).
A subclass of the python Unpickler that implements unpickling of _ElementTree types.
def col
Given any list, array, or matrix, return a 1-column matrix.
def isfloat
Matches ANY number; it can be a decimal, scientific notation, what have you CAUTION - this will also ...
def wq_wait
This function waits until the work queue is completely empty.
def statisticalInefficiency
Compute the (cross) statistical inefficiency of (two) timeseries.
def flat
Given any list, array, or matrix, return a single-index array.
def isdecimal
Matches things with a decimal only; see isint and isfloat.
def invert_svd
Invert a matrix using singular value decomposition.
def wq_wait1
This function waits ten seconds to see if a task in the Work Queue has finished.
def lp_load
Use this instead of pickle.load for unpickling anything that contains _ElementTree types...
def queue_up_src_dest
Submit a job to the Work Queue.
def MissingFileInspection
def printcool
Cool-looking printout for slick formatting of output.
def pvec1d
Printout of a 1-D vector.
def concurrent_map
Similar to the bultin function map().
def isint
ONLY matches integers! If you have a decimal point? None shall pass!
def lp_dump
Use this instead of pickle.dump for pickling anything that contains _ElementTree types.
def row
Given any list, array, or matrix, return a 1-row matrix.
def floatornan
Returns a big number if we encounter NaN.
def warn_once
Prints a warning but will only do so once in a given run.