#!/usr/bin/env python from optparse import OptionParser import sys import os import os.path import pty import re import select import subprocess parser = OptionParser(usage="%prog [options] DIR/FILE", version="%prog $Id: $", description="Call LaTeX and friends to typeset all tex files in DIR/FILE. By default up-to-date files are skipped. Care is taken to run all required commands until the result is stable. By default, pdflatex is used, but use of pstricks forces dvi->ps->pdf. The utility program rmligs ist used to improve output for german language texts. Use \"texall -Sqfi batchmode file.tex\" to simulate the behaviour of (pdf)latex in non-interactive mode (useful for emacs).") parser.add_option("-n", "--dry-run", action="store_false", dest="act", default=True, help="do not run any program") parser.add_option("-f", "--force", action="store_true", dest="force", default=False, help="regenerate up-to-date files") parser.add_option("-N", "--non-recursive", action="store_false", dest="recurse", default=True, help="disable recursion into subdirectories") parser.add_option("-s", "--summary", dest="summary", help="summarise failures and processed files at the end of the run (one of failures (default), files, both, no)", choices=("no","files","failures","both"), default="failures") parser.add_option("-v", "--verbose", action="count", dest="verbosity", default=1, help="explain what is going on") parser.add_option("-q", "--quiet", action="store_const", dest="verbosity", const=0, help="suppress progress reports") parser.add_option("-o", "--show-output", action="store_true", dest="showoutput", default=False, help="show the output of the called programs") parser.add_option("-i", "--interaction", dest="interactionmode", choices=("batchmode", "nonstopmode", "scrollmode", "errorstopmode"), help="control the behaviour of latex (on of batchmode (default), nonstopmode (default for -o), scrollmode, errorstopmode)") parser.add_option("-I", "--ignore", action="store", dest="ignorepattern", default=".*vorlage.*|\.\#.*", help="regular expression of filenames to ignore") parser.add_option("-P", "--required-pattern", action="store", dest="requiredpattern", default='^\\\\documentclass|%%% TeX-master: t', help="regular expression that must match the file content") parser.add_option("-T", "--preserve-tempfiles", action="store_false", dest="deletetempfiles", default=True, help="preserve temporary files (default: delete)") parser.add_option("-S", "--single-run", action="store_true", dest="singlerun", default=False, help="stop after the first LaTeX run, no matter if finished") (options, args) = parser.parse_args() if options.interactionmode == None: if options.showoutput: options.interactionmode = "nonstopmode" else: options.interactionmode = "batchmode" re_ignore = None if options.ignorepattern != "": re_ignore = re.compile(options.ignorepattern) re_required = re.compile(options.requiredpattern) re_texfile = re.compile(".*\.tex$", re.I) errors = [] procfiles = [] errstr = {False: "ERROR", True: "WARNING"} def error(file, desc, warning=False): msg = "%s: %s: %s"%(errstr[warning], os.path.normpath(file), desc) if options.verbosity: print msg errors.append(msg) if os.path.exists("/dev/null"): null = file("/dev/null", "wb") else: null = file("nul:", "wb") path = os.path.expandvars("$PATH") if path == "$PATH": path = os.path.defpath path = path.split(os.path.pathsep) def isinpath(name): for p in path: if os.path.isfile(os.path.join(p, name)): return True return False if isinpath("rmligs"): rmligs_name = "rmligs" elif isinpath("rmligs-german"): rmligs_name = "rmligs-german" else: rmligs_name = "" error("rmligs", "Command not found. Please install rmligs (or rmligs-german) for optimal results", warning=True) # iterate over all .tex files that are not ignored by -i or -p def alltexfiles(args): for a in args: if os.path.isdir(a): for dir, subdirs, files in os.walk(a): if not options.recurse: subdirs[:] = [] for name in files: if re_texfile.match(name): if re_ignore.match(name): if options.verbosity > 2: print "%s: skipped because of ignored name"%os.path.normpath(os.path.join(dir,name)) continue for m in texgrep(re_required, dir, name): yield (dir, name) break else: if options.verbosity > 2: print "%s: skipped because no main latex file"%os.path.normpath(os.path.join(dir,name)) elif os.path.isfile(a): if re_texfile.match(a): (dirname, filename) = os.path.split(a) if dirname == "": dirname = "." yield (dirname, filename) else: error(a, "is no .tex file; skipped") elif os.path.isfile(a+".tex"): (dirname, filename) = os.path.split(a+".tex") if dirname == "": dirname = "." yield(dirname, filename) else: error(a, "file not found") def texgrep(matcher, dirname, filename, recurse=False): try: f = open(os.path.join(dirname,filename)) content = f.read() for m in matcher.finditer(content): yield m if recurse: for dep in dependencies(dirname, filename, texonly=True): for m in texgrep(matcher, "", dep, recurse=False): yield m except IOError, (errno, strerror): error(os.path.join(dirname, filename), "could not be read: %s"%strerror) except StopIteration: f.close() else: f.close() kpsepaths={} def texpath(dirname,filename,pathtype="tex",progname="latex",extlist=[],allowsystemfiles=False): if not kpsepaths.has_key((pathtype,progname)): kpse=subprocess.Popen(["kpsepath", "-n", progname, pathtype], stdout=subprocess.PIPE) (out,err)=kpse.communicate() if kpse.wait() != 0: error("kpsepath", "failed to get paths for %s"%pathtype) kpsepaths[(pathtype,progname)] = out[:-1].split(":") if os.path.isfile(os.path.join(dirname,filename)): return os.path.join(dirname,filename) for ext in extlist: if os.path.isfile(os.path.join(dirname,filename+ext)): return os.path.join(dirname,filename+ext) RECURSIONDEPTH=10 re_inputinclude = re.compile('\\\\(input|include|@input)\\{([^}]*)\\}') re_usepackage = re.compile('\\\\usepackage(\\[.*?\\])?\\{([^}]*)\\}') re_graphics = re.compile('\\\\includegraphics(\\[.*?\\])?\\{([^}]*)\\}') graphics_ext = ["", ".pdf", ".eps", ".png", ".jpg"] re_bibliography = re.compile('\\\\bibliography\\{([^}]*)\\}') re_commawhitespace = re.compile('\\s*,\\s*') def dependencies(dirname, filename, texonly=False, recurse=RECURSIONDEPTH): if not recurse: return try: f = open(os.path.join(dirname,filename)) content = f.read() for m in re_inputinclude.finditer(content): texs = m.group(2) for tex in re_commawhitespace.split(texs): if not os.path.isfile(os.path.join(dirname,tex)): tex+=".tex" if os.path.isfile(os.path.join(dirname,tex)): yield os.path.normpath(os.path.join(dirname,tex)) for m in dependencies(dirname, tex, texonly, recurse-1): yield m for m in re_usepackage.finditer(content): stys = m.group(2) for sty in re_commawhitespace.split(stys): if not os.path.isfile(os.path.join(dirname,sty)): sty+=".sty" if os.path.isfile(os.path.join(dirname,sty)): yield os.path.normpath(os.path.join(dirname,sty)) for m in dependencies(dirname, sty, texonly, recurse-1): yield m if not texonly: for m in re_graphics.finditer(content): names = m.group(2) for name in re_commawhitespace.split(names): for ext in graphics_ext: if os.path.isfile(os.path.join(dirname,name+ext)): yield os.path.normpath(os.path.join(dirname,name+ext)) for m in re_bibliography.finditer(content): bibs = m.group(1) for bib in re_commawhitespace.split(bibs): if bib[-4:] != ".bib": bib += ".bib" if bib[-8:] != "-blx.bib": yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib))) except IOError, (errno, strerror): error(os.path.join(dirname, filename), "could not be read: %s"%strerror) except StopIteration: f.close() else: f.close() def bibfiles(dirname, texname): for m in texgrep(re_bibliography, dirname, texname, recurse=True): bibs = m.group(1) for bib in re_commawhitespace.split(bibs): if bib[-4:] != ".bib": bib += ".bib" if bib[-8:] != "-blx.bib": yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib))) def outdatedtexfiles(args): for dirname, texname in alltexfiles(args): #strip .tex extension jobname=texname[:-4] pdfname=jobname+".pdf" reason = "" # check if an update is needed: if options.force and not options.verbosity: yield (dirname, texname, reason) elif not os.path.isfile(os.path.join(dirname,pdfname)): if options.verbosity: reason = " (because .pdf does not exist)" yield (dirname, texname, reason) else: pdftime = os.path.getmtime(os.path.join(dirname, pdfname)) if pdftime < os.path.getmtime(os.path.join(dirname, texname)): if options.verbosity: reason = " (because .tex is newer than .pdf)" yield (dirname, texname, reason) else: for f in dependencies(dirname, texname): if not os.path.isfile(f): error(os.path.join(dirname,texname), "dependency not found: %s"%f, warning=True) continue if pdftime < os.path.getmtime(f): if options.verbosity: reason = " (because %s is newer than .pdf)"%f yield (dirname, texname, reason) break else: r = haderrors(dirname, jobname) if r: if options.verbosity: reason = r yield (dirname, texname, reason) elif options.force: reason = " (because of --force)" yield (dirname, texname, reason) elif options.verbosity > 2: print "%s: skipped because up-to-date"%os.path.normpath(os.path.join(dirname, texname)) re_nopdftex = re.compile('\\\\usepackage(\\[.*?\\])?\\{[^}]*pstricks[^}]*\\}') def detecttextype(dirname, texname): for m in texgrep(re_nopdftex, dirname, texname, recurse=True): return "latex" return "pdflatex" re_rmligs = re.compile('\\\usepackage((\\[.*?\\])?\\{n?german\\}|\\[[^]]*?german[^]]*?\\]\\{babel\\})') def detectrmligs(dirname, texname): if rmligs_name == "": return False for m in texgrep(re_rmligs, dirname, texname, recurse=True): return True return False MAXRUNS=5 def processtexfiles(args): for dirname, texname, reason in outdatedtexfiles(args): procfiles.append(os.path.normpath(os.path.join(dirname,texname))) if options.verbosity: print "processing %s%s..."%(os.path.normpath(os.path.join(dirname,texname)),reason) # list of temporary files to be removed later tmplist = [] # support for pstricks/plain latex: tex = detecttextype(dirname, texname) # detect usage of german/ngerman and generate temporary files with rmlig realname = texname if detectrmligs(dirname, texname): realname = preparermligs(texname, dirname, tmplist) runtex(tex, texname, realname, dirname) if not options.singlerun: #strip .tex extension jobname=texname[:-4] # run bibtex if any bibfile changed: for bib in bibfiles(dirname, texname): bbl = os.path.normpath(os.path.join(dirname, jobname+".bbl")) if options.force or not os.path.isfile(bbl) or os.path.getmtime(bbl) < os.path.getmtime(bib): reason = "" if options.verbosity: if not os.path.isfile(bbl): reason = " (because .bbl does not exist yet)" elif os.path.getmtime(bbl) < os.path.getmtime(bib): reason = " (because %s is newer than .bbl)"%bib else: reason = " (because of --force)" run(["bibtex8", "--wolfgang", jobname], dirname, reason=reason) runtex(tex, texname, realname, dirname, reason=" (because of updated .bbl)") break # check for undefined references and run requests numrun=0 reqs=options.act while reqs: reqs = requests(os.path.join(dirname, jobname+".log")) pri = reqs.values() pri.sort(reverse=True) for p in pri: for req in reqs.keys(): rp = reqs[req] if rp == p: reason = "" if options.verbosity: reason = " (because of request in .aux file, priority %d)"%rp if req == "latex": runtex(tex, texname, realname, dirname, reason=reason) elif req == "bibtex": run(["bibtex8", "--wolfgang", jobname], dirname, reason=reason) else: error(os.path.join(dirname,jobname+".aux"), "unsupported request: %s"%req) numrun+=1 if numrun==MAXRUNS: error(os.path.join(dirname, texname), "does not stabilise after %i runs"%MAXRUNS) break # update index if it exists - no way of knowing if it was updated idx = jobname + ".idx" if os.path.isfile(os.path.join(dirname, idx)): run(["makeindex", jobname], dirname, reason=" (because .idx file might have changed)") runtex(tex, texname, realname, dirname, reason=" (because .ind file might have changed)") if tex == "latex": run(["dvips", jobname+".dvi"], dirname) run(["ps2pdf", jobname+".ps"], dirname) rmtempfile(tmplist, dirname) re_logmatcher = re.compile("^LaTeX Warning: There were undefined references\.$|^LaTeX Warning: Label\(s\) may have changed\. Rerun to get cross-references right\.$|^REQ:(\d+):(\w+):") def requests(logpath): reqs = {} def addrequest(name, priority): if reqs.has_key(name): reqs[name] = max(reqs[name], priority) else: reqs[name] = priority try: log = open(logpath) for line in log: m = re_logmatcher.match(line) if m: if m.group() == 'LaTeX Warning: There were undefined references.': addrequest("latex",0) elif m.group() == 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.': addrequest("latex",0) elif m.group(1)!=None: addrequest(m.group(2), int(m.group(1))) # TODO: add support for multiple bibliographies, see biblatex.pdf Sec. 2.4.4 except IOError, (errno, strerror): error(logpath, "could not be read: %s"%strerror) else: log.close() return reqs re_error = re.compile('^! ') def haderrors(dirname, jobname): logpath = os.path.join(dirname, jobname+".log") if not os.path.isfile(logpath): return None try: f = open(logpath) for line in f: if re_error.match(line): return " (because of error in .log file)" if re_logmatcher.match(line): return " (because of request in .log file)" except IOError, (errno, strerror): error(logpath, "could not be read: %s"%strerror) else: f.close() return False def rmtempfile(filename, dirname): if options.deletetempfiles: if not isinstance(filename, list): filename = [filename] if options.verbosity > 1: print " removing %s"%(", ".join(filename)) if options.act: for f in filename: fullname = os.path.join(dirname, f) if os.path.isfile(fullname): os.remove(fullname) else: error(fullname, "could not be removed (does not exist)", warning=True) def run(arglist, dirname, reason="", inf=None, outf=None, stderr=None): """Run a command with optional io redirections. arglist -- list of progam name and arguments dirname -- the directory the command will be executed in reason -- reason for running this command, used in progress output inf -- file to be used as stdin, or alternativly a filter function for user input outf -- file to be used as stdout, or alternativly a filter function for program output stderr -- file to be used as stderr """ if options.verbosity: print " running %s%s..."%(" ".join(arglist), reason) if options.act: master, slave = None, None if isinstance(inf, file): stdin = inf inf = None elif options.showoutput and not subprocess.mswindows: master, slave = pty.openpty() master = os.fdopen(master, "rw") stdin=slave elif inf or options.showoutput: stdin=subprocess.PIPE else: stdin=null if isinstance(outf, file): stdout = outf outf = None elif options.showoutput and not subprocess.mswindows: if slave == None: master, slave = pty.openpty() master = os.fdopen(master, "rw") stdout=slave elif outf or options.showoutput: stdout=subprocess.PIPE else: stdout=null try: proc = subprocess.Popen(arglist, stdin=stdin, stdout=stdout, stderr=stderr, cwd=dirname) if stdin == slave and stdin != None: childin = master else: childin = proc.stdin if stdout == slave and stdout != None: childout = master else: childout = proc.stdout selectlist = [] outlist = [] if childin != None: selectlist.append(sys.stdin) if childout != None: selectlist.append(childout) outlist.append(childout) while proc.poll() == None or select.select(outlist, [], [], 0)[0]: ready = select.select(selectlist, [], [], 0.1)[0] for f in ready: if f == childout: data = os.read(childout.fileno(), 1024) if data == "": # EOF outlist.remove(childout) selectlist.remove(childout) continue if outf: data = outf(data) if data != None: sys.stdout.write(data) sys.stdout.flush() if f == sys.stdin: if proc.poll() != None: # process terminated, does not take input anymore selectlist.remove(sys.stdin) continue data = os.read(sys.stdin.fileno(), 1024) if data == "": # EOF selectlist.remove(sys.stdin) continue if inf: data = inf(data) if data != None: childin.write(data) childin.flush() ret = proc.wait() if ret: error(dirname, "failed command: %s returned %i"%(" ".join(arglist), ret)) except (OSError, IOError), e: error(dirname, "failed command: %s: %s"%(" ".join(arglist),str(e))) def preparermligs(texname, dirname, tmplist): jobname=texname[:-4] try: texfile = open(os.path.join(dirname, texname)) except IOError, (errno, strerror): error(os.path.join(dirname, texname), "could not be read: %s. Not using rmligs."%strerror) return texname else: try: (subdir, texbasename) = os.path.split(texname) rmligsbasename = ".%s.rmligs.tex"%texbasename[:-4] rmligsname = os.path.join(subdir, rmligsbasename) if options.act: rmligsfile = open(os.path.join(dirname, rmligsname), "w") else: rmligsfile = null except IOError, (errno, strerror): error(os.path.join(dirname, rmligsname), "could not be written: %s. Not using rmligs."%strerror) return texname else: tmplist.append(rmligsname) def rewriteinputrmligs(m): name=m.group(2) if not os.path.isfile(os.path.join(dirname,name)): name+=".tex" if os.path.isfile(os.path.join(dirname, name)): name = preparermligs(name, dirname, tmplist) else: error(os.path.join(dirnam, texname), "%s: File could not be located. Not using rmligs."%name, warning=True) return "\\%s{%s}"%(m.group(1), name[:-4]) def filteroutput(d): rmligsfile.write(re_inputinclude.sub(rewriteinputrmligs, d)) return None run([rmligs_name, "-f"], dirname, inf=texfile, outf=filteroutput) texfile.close() if rmligsfile is not null: rmligsfile.close() return rmligsname re_rmligsfile = re.compile("\\.([^/ ]+)\\.rmligs\\.tex") def runtex(tex, texname, realname, dirname, reason=""): if not options.showoutput: outf = None else: def outf(d): return re_rmligsfile.sub("\\1.tex", d) if texname != realname: run([tex, "-interaction", options.interactionmode, "-jobname", texname[:-4], realname], dirname, outf=outf, reason=reason) else: run([tex, "-interaction", options.interactionmode, texname], dirname, outf=outf, reason=reason) # main program: try: if args: processtexfiles(args) else: processtexfiles(["."]) if options.summary in [ "files", "both" ] : if procfiles: print "Processed the following files:" for f in procfiles: print " %s"%f else: print "Processed no files." if options.summary in [ "failures", "both" ] : if errors: print "The following problems occured:" for f in errors: print " %s"%f if errors: sys.exit(1) except KeyboardInterrupt: sys.exit(2)