path: root/texall

#!/usr/bin/env python

from optparse import OptionParser
import sys
import os
import os.path
import pty
import re
import select
import subprocess
import fnmatch
import dircache

parser = OptionParser(usage="%prog [options] DIR/FILE",
                      version="%prog $Id: $",
                      description="Call LaTeX and friends to typeset all tex files in DIR/FILE. By default up-to-date files are skipped. Care is taken to run all required commands until the result is stable. By default, pdflatex is used, but use of pstricks forces dvi->ps->pdf. The utility program rmligs ist used to improve output for german language texts. Use \"texall -Sqfi batchmode file.tex\" to simulate the behaviour of (pdf)latex in non-interactive mode (useful for emacs).")
parser.add_option("-n", "--dry-run",
                  action="store_false", dest="act", default=True,
                  help="do not run any program")
parser.add_option("-f", "--force",
                  action="store_true", dest="force", default=False,
                  help="regenerate up-to-date files")
parser.add_option("-N", "--non-recursive",
                  action="store_false", dest="recurse", default=True,
                  help="disable recursion into subdirectories")
parser.add_option("-s", "--summary", dest="summary",
                  help="summarise failures and processed files at the end of the run (one of failures (default), files, both, no)",
                  choices=("no","files","failures","both"), default="failures")
parser.add_option("-v", "--verbose",
                  action="count", dest="verbosity", default=1,
                  help="explain what is going on")
parser.add_option("-q", "--quiet",
                  action="store_const", dest="verbosity", const=0,
                  help="suppress progress reports")
parser.add_option("-o", "--show-output",
                  action="store_true", dest="showoutput", default=False,
                  help="show the output of the called programs")
parser.add_option("-i", "--interaction",
                  dest="interactionmode", choices=("batchmode", "nonstopmode", "scrollmode", "errorstopmode"),
                  help="control the behaviour of latex (on of batchmode (default), nonstopmode (default for -o), scrollmode, errorstopmode)")
parser.add_option("-I", "--ignore",
                  action="store", dest="ignorepattern", default=".*vorlage.*|\.\#.*|\..*\.rmligs\.tex",
                  help="regular expression of filenames to ignore")
parser.add_option("-P", "--required-pattern",
                  action="store", dest="requiredpattern", default='^\\\\documentclass|%%% TeX-master: t',
                  help="regular expression that must match the file content")
parser.add_option("-T", "--preserve-tempfiles",
                  action="store_false", dest="deletetempfiles", default=True,
                  help="preserve temporary files (default: delete)")
parser.add_option("-S", "--single-run",
                  action="store_true", dest="singlerun", default=False,
                  help="stop after the first LaTeX run, no matter if finished")
(options, args) = parser.parse_args()

if options.interactionmode == None:
    if options.showoutput:
        options.interactionmode = "nonstopmode"
    else:
        options.interactionmode = "batchmode"

re_ignore = None
if options.ignorepattern != "":
    re_ignore = re.compile(options.ignorepattern)

re_required = re.compile(options.requiredpattern)

re_texfile = re.compile(".*\.tex$", re.I)

errors = []
procfiles = []

errstr = {False: "ERROR", True: "WARNING"}
def error(file, desc, warning=False):
    msg = "%s: %s: %s"%(errstr[warning], os.path.normpath(file), desc)
    if options.verbosity:
        print msg
    errors.append(msg)

null = file(os.path.devnull, "wb")

# platform checking:
iswindows = (sys.platform == "win32")

# provide a copy of os.path.relpath from python 2.6
if not iswindows:
    def relpath(path, start=os.path.curdir):
        """Return a relative version of a path"""
        
        start_list = os.path.abspath(start).split(os.path.sep)
        path_list = os.path.abspath(path).split(os.path.sep)
    
        # Work out how much of the filepath is shared by start and path.
        i = len(os.path.commonprefix([start_list, path_list]))

        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
        return os.path.join(*rel_list)
else:
    def relpath(path, start=os.path.curdir):
        """Return a relative version of a path"""

        start_list = os.path.abspath(start).split(os.path.sep)
        path_list = os.path.abspath(path).split(os.path.sep)
        if start_list[0].lower() != path_list[0].lower():
            unc_path, rest = os.path.splitunc(path)
            unc_start, rest = os.path.splitunc(start)
            if bool(unc_path) ^ bool(unc_start):
                raise ValueError("Cannot mix UNC and non-UNC paths (%s and %s)"
                                                                % (path, start))
            else:
                raise ValueError("path is on drive %s, start on drive %s"
                                 % (path_list[0], start_list[0]))
        # Work out how much of the filepath is shared by start and path.
        for i in range(min(len(start_list), len(path_list))):
            if start_list[i].lower() != path_list[i].lower():
                break
        else:
            i += 1

        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
        return os.path.join(*rel_list)

path = os.path.expandvars("$PATH")
if path == "$PATH":
    path = os.path.defpath
path = path.split(os.path.pathsep)
def isinpath(name):
    for p in path:
        if os.path.isfile(os.path.join(p, name)):
            return True
    return False

if isinpath("rmligs"):
    rmligs_name = "rmligs"
elif isinpath("rmligs-german"):
    rmligs_name = "rmligs-german"
else:
    rmligs_name = ""
    error("rmligs", "Command not found. Please install rmligs (or rmligs-german) for optimal results", warning=True)

# iterate over all .tex files that are not ignored by -i or -p
def alltexfiles(args):
    for a in args:
        if os.path.isdir(a):
            for dir, subdirs, files in os.walk(a):
                if not options.recurse:
                    subdirs[:] = []
                for name in files:
                    if re_texfile.match(name):
                        if re_ignore.match(name):
                            if options.verbosity > 2:
                                print "%s: skipped because of ignored name"%os.path.normpath(os.path.join(dir,name))
                            continue
                        for m in texgrep(re_required, dir, name):
                            yield (dir, name)
                            break
                        else:
                            if options.verbosity > 2:
                                print "%s: skipped because no main latex file"%os.path.normpath(os.path.join(dir,name))
        elif os.path.isfile(a):
            if re_texfile.match(a):
                (dirname, filename) = os.path.split(a)
                if dirname == "":
                    dirname = "."
                yield (dirname, filename)
            else:
                error(a, "is no .tex file; skipped")
        elif os.path.isfile(a+".tex"):
            (dirname, filename) = os.path.split(a+".tex")
            if dirname == "":
                dirname = "."
            yield(dirname, filename)
        else:
            error(a, "file not found")

def texgrep(matcher, dirname, filename, recurse=False):
    try:
        f = open(os.path.join(dirname,filename))
        content = f.read()
        for m in matcher.finditer(content):
            yield m
        if recurse:
            for dep in dependencies(dirname, filename, texonly=True):
                for m in texgrep(matcher, "", dep, recurse=False):
                    yield m
    except IOError, (errno, strerror):
        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
    except StopIteration:
        f.close()
    else:
        f.close()

kpsepaths={}
def texpath(dirname,filename,pathtype="tex",progname="latex",extlist=[],allowsystemfiles=False):
    if not kpsepaths.has_key((pathtype,progname)):
        kpse=subprocess.Popen(["kpsepath", "-n", progname, pathtype], stdout=subprocess.PIPE)
        (out,err)=kpse.communicate()
        if kpse.wait() != 0:
            error("kpsepath", "failed to get paths for %s"%pathtype)
        kpsepaths[(pathtype,progname)] = out[:-1].split(":")
    if os.path.isfile(os.path.join(dirname,filename)):
        return os.path.join(dirname,filename)
    for ext in extlist:
        if os.path.isfile(os.path.join(dirname,filename+ext)):
            return os.path.join(dirname,filename+ext)

RECURSIONDEPTH=10
re_inputinclude = re.compile('\\\\(input|include|@input)\\{([^}]*)\\}')
re_usepackage = re.compile('\\\\usepackage(\\[.*?\\])?\\{([^}]*)\\}')
re_graphics = re.compile('\\\\includegraphics(\\[.*?\\])?\\{([^}]*)\\}')
graphics_ext = ["", ".pdf", ".eps", ".png", ".jpg"]
re_bibliography = re.compile('\\\\bibliography\\{([^}]*)\\}')
re_commawhitespace = re.compile('\\s*,\\s*')
def dependencies(dirname, filename, texonly=False, recurse=RECURSIONDEPTH):
    if not recurse:
        return
    try:
        f = open(os.path.join(dirname,filename))
        content = f.read()
        for m in re_inputinclude.finditer(content):
            texs = m.group(2)
            for tex in re_commawhitespace.split(texs):
                if not os.path.isfile(os.path.join(dirname,tex)):
                    tex+=".tex"
                if os.path.isfile(os.path.join(dirname,tex)):
                    yield os.path.normpath(os.path.join(dirname,tex))
                    for m in dependencies(dirname, tex, texonly, recurse-1):
                        yield m
        for m in re_usepackage.finditer(content):
            stys = m.group(2)
            for sty in re_commawhitespace.split(stys):
                if not os.path.isfile(os.path.join(dirname,sty)):
                    sty+=".sty"
                if os.path.isfile(os.path.join(dirname,sty)):
                    yield os.path.normpath(os.path.join(dirname,sty))
                    for m in dependencies(dirname, sty, texonly, recurse-1):
                        yield m
        if not texonly:
            for m in re_graphics.finditer(content):
                names = m.group(2)
                for name in re_commawhitespace.split(names):
                    for ext in graphics_ext:
                        if os.path.isfile(os.path.join(dirname,name+ext)):
                            yield os.path.normpath(os.path.join(dirname,name+ext))
            for m in re_bibliography.finditer(content):
                bibs = m.group(1)
                for bib in re_commawhitespace.split(bibs):
                    if bib[-4:] != ".bib":
                        bib += ".bib"
                    if bib[-8:] != "-blx.bib":
                        yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib)))
                    
    except IOError, (errno, strerror):
        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
    except StopIteration:
        f.close()
    else:
        f.close()

def bibfiles(dirname, texname):
    for m in texgrep(re_bibliography, dirname, texname, recurse=True):
        bibs = m.group(1)
        for bib in re_commawhitespace.split(bibs):
            if bib[-4:] != ".bib":
                bib += ".bib"
            if bib[-8:] != "-blx.bib":
                yield bib

def outdatedtexfiles(args):
    for dirname, texname in alltexfiles(args):
        #strip .tex extension
        jobname=texname[:-4]
        pdfname=jobname+".pdf"
        reason = ""
        # check if an update is needed:
        if options.force and not options.verbosity:
            yield (dirname, texname, reason)
        elif not os.path.isfile(os.path.join(dirname,pdfname)):
            if options.verbosity:
                reason = " (because .pdf does not exist)"
            yield (dirname, texname, reason)
        else:
            pdftime = os.path.getmtime(os.path.join(dirname, pdfname))
            if pdftime < os.path.getmtime(os.path.join(dirname, texname)):
                if options.verbosity:
                    reason = " (because .tex is newer than .pdf)"
                yield (dirname, texname, reason)
            else:
                for f in dependencies(dirname, texname):
                    if not os.path.isfile(f):
                        error(os.path.join(dirname,texname), "dependency not found: %s"%f, warning=True)
                        continue
                    if pdftime < os.path.getmtime(f):
                        if options.verbosity:
                            reason = " (because %s is newer than .pdf)"%f
                        yield (dirname, texname, reason)
                        break
                else:
                    r = haderrors(dirname, jobname)
                    if r:
                        if options.verbosity:
                            reason = r
                        yield (dirname, texname, reason)
                    elif options.force:
                        reason = " (because of --force)"
                        yield (dirname, texname, reason)
                    elif options.verbosity > 2:
                        print "%s: skipped because up-to-date"%os.path.normpath(os.path.join(dirname, texname))

re_nopdftex = re.compile('\\\\usepackage(\\[.*?\\])?\\{[^}]*pstricks[^}]*\\}')
def detecttextype(dirname, texname):
    for m in texgrep(re_nopdftex, dirname, texname, recurse=True):
        return "latex"
    return "pdflatex"

re_rmligs = re.compile('\\\usepackage((\\[.*?\\])?\\{n?german\\}|\\[[^]]*?german[^]]*?\\]\\{babel\\})')
def detectrmligs(dirname, texname):
    if rmligs_name == "":
        return False
    for m in texgrep(re_rmligs, dirname, texname, recurse=True):
        return True
    return False

def bblfiles(dirname, jobname):
    files = dircache.listdir(dirname)
    return fnmatch.filter(files, "%s*.bbl"%jobname)

MAXRUNS=5
def processtexfiles(args):
    for dirname, texname, reason in outdatedtexfiles(args):
        procfiles.append(os.path.normpath(os.path.join(dirname,texname)))
        if options.verbosity:
            print "processing %s%s..."%(os.path.normpath(os.path.join(dirname,texname)),reason)

        # list of temporary files to be removed later
        tmplist = []        

        # support for pstricks/plain latex:
        tex = detecttextype(dirname, texname)

        # detect usage of german/ngerman and generate temporary files with rmlig
        realname = texname
        if detectrmligs(dirname, texname):
            realname = preparermligs(texname, dirname, tmplist)

        #strip .tex extension
        jobname=texname[:-4]

        # remove all outdated .bbl files (will be recreated by requests)
        bbls = bblfiles(dirname, jobname)
        bblname = jobname+".bbl"
        for bib in bibfiles(dirname, texname):
            bibpath = os.path.join(dirname, os.path.expanduser(bib))
            bibtime = os.path.getmtime(bibpath)
            nextbbls = bbls[:]
            for bbl in bbls:
                bblpath = os.path.normpath(os.path.join(dirname, bbl))
                bbltime = os.path.getmtime(bblpath)
                if options.force or bbltime < bibtime:
                    reason = ""
                    if options.verbosity:
                        if bbltime < bibtime:
                            reason = " (because %s is newer than %s)"%(bib,bbl)
                        else:
                            reason = " (because of --force)"
                    rmtempfile(bbl, dirname, force=True, reason=reason)
                    nextbbls.remove(bbl)
            bbls = nextbbls

        runtex(tex, texname, realname, dirname)

        if not options.singlerun:
            # check for undefined references and run requests
            numrun=0
            reqs=options.act
            while reqs:
                reqs = requests(os.path.join(dirname, jobname+".log"))
                if not reqs: break
                pri = reqs.values()
                pri.sort(reverse=True)
                lastp = pri[0] + 1
                for p in pri:
                    if p == lastp: continue
                    lastp = p
                    for req in reqs.keys():
                        rp = reqs[req]
                        if rp == p:
                            reason = ""
                            if options.verbosity:
                                reason = " (because of request in .aux file, priority %d)"%rp
                            if req == "latex":
                                runtex(tex, texname, realname, dirname, reason=reason)
                            elif isinstance(req, tuple):
                                run(req, dirname, reason)
                            else:
                                error(os.path.join(dirname,jobname+".aux"), "unsupported request: %s"%req)
                numrun+=1
                if numrun==MAXRUNS:
                    error(os.path.join(dirname, texname), "does not stabilise after %i runs"%MAXRUNS)
                    break

            # update index if it exists - no way of knowing if it was updated
            idx = jobname + ".idx"
            if os.path.isfile(os.path.join(dirname, idx)):
                run(["makeindex", jobname], dirname, reason=" (because .idx file might have changed)")
                runtex(tex, texname, realname, dirname, reason=" (because .ind file might have changed)")

        if tex == "latex":
            run(["dvips", jobname+".dvi"], dirname)
            run(["ps2pdf", jobname+".ps"], dirname)

        rmtempfile(tmplist, dirname)

re_logmatcher = re.compile("^LaTeX Warning: There were undefined references\.$|^LaTeX Warning: Label\(s\) may have changed\. Rerun to get cross-references right\.$|^REQ:(\d+):(\w+):REQ|^REQ:(\d+):(\w+):$")
def requests(logpath):
    reqs = {}
    def addrequest(name, priority):
        if reqs.has_key(name):
            reqs[name] = max(reqs[name], priority)
        else:
            reqs[name] = priority
    try:
        log = open(logpath)
        inrequest = False
        for line in log:
            if inrequest:
                # strip newline character
                while line[-1] in ("\n", "\r"):
                    line = line[:-1]
                if line[:7] == "binary=":
                    reqbinary = line[7:]
                elif line[:7] == "option=":
                    reqoptions.extend(line[7:].split(" "))
                elif line[:7] == "infile=":
                    reqinfile = line[7:]
                elif line == ":REQ":
                    if reqbinary != None:
                        if reqbinary not in ("bibtex", "bibtex8"):
                            error(logpath, "ignoring request to run %s for security reasons"%reqbinary, warning=True)
                            inrequest = False
                            continue
                        reqoptions[:0] = [reqbinary]
                        if reqbinary == "bibtex8":
                            reqoptions[1:1] = [ "--wolfgang" ]
                    if reqinfile != None:
                        reqoptions.append(reqinfile)
                    addrequest(tuple(reqoptions), reqpri)
                    inrequest=False
                else:
                    error(logpath, "malformatted request ignored", warning=True)
                    inrequest=False
            else:
                m = re_logmatcher.match(line)
                if m:
                    if m.group() == 'LaTeX Warning: There were undefined references.':
                        addrequest("latex",0)
                    elif m.group() == 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.':
                        addrequest("latex",0)
                    elif m.group(1)!=None:
                        addrequest(m.group(2), int(m.group(1)))
                    elif m.group(3)!=None:
                        inrequest = True
                        reqname = m.group(4)
                        reqpri = int(m.group(3))
                        reqbinary = None
                        reqoptions = []
                        reqinfile = None
    except IOError, (errno, strerror):
        error(logpath, "could not be read: %s"%strerror)
    else:
        log.close()
    return reqs

re_error = re.compile('^! ')
def haderrors(dirname, jobname):
    logpath = os.path.join(dirname, jobname+".log")
    if not os.path.isfile(logpath):
        return None
    try:
        f = open(logpath)
        for line in f:
            if re_error.match(line):
                return " (because of error in .log file)"
            if re_logmatcher.match(line):
                return " (because of request in .log file)"
    except IOError, (errno, strerror):
        error(logpath, "could not be read: %s"%strerror)
    else:
        f.close()
    return False

def rmtempfile(filename, dirname, force=False, reason=""):
    if options.deletetempfiles:
        if not isinstance(filename, list):
            filename = [filename]
        if options.verbosity > 1:
            print "  removing %s%s"%(", ".join(filename), reason)
        if options.act:
            for f in filename:
                fullname = os.path.join(dirname, f)
                if os.path.isfile(fullname):
                    os.remove(fullname)
                else:
                    error(fullname, "could not be removed (does not exist)", warning=True)

def run(arglist, dirname, reason="", inf=None, outf=None, stderr=None):
    """Run a command with optional io redirections.

    arglist -- list of progam name and arguments
    dirname -- the directory the command will be executed in
    reason -- reason for running this command, used in progress output
    inf -- file to be used as stdin, or alternativly a filter function for user input
    outf -- file to be used as stdout, or alternativly a filter function for program output
    stderr -- file to be used as stderr
    """
    if options.verbosity:
        redir =""
        if hasattr(inf, "name") and inf.name != "" and inf.name[0] != "<":
            name = relpath(inf.name, dirname)
            redir = " < %s"%name
        print "  running %s%s%s..."%(" ".join(arglist), redir, reason)
        
    if options.act:
        master, slave = None, None
        
        if isinstance(inf, file):
            stdin = inf
            inf = None
        elif options.showoutput and not iswindows:
            master, slave = pty.openpty()
            master = os.fdopen(master, "rw")
            stdin=slave
        elif inf or options.showoutput:
            stdin=subprocess.PIPE
        else:
            stdin=null

        if isinstance(outf, file):
            stdout = outf
            outf = None
        elif options.showoutput and not iswindows:
            if slave == None:
                master, slave = pty.openpty()
                master = os.fdopen(master, "rw")
            stdout=slave
        elif outf or options.showoutput:
            stdout=subprocess.PIPE
        else:
            stdout=null

        try:
            proc = subprocess.Popen(arglist, stdin=stdin, stdout=stdout, stderr=stderr, cwd=dirname)

            if stdin == slave and stdin != None:
                childin = master
            else:
                childin = proc.stdin
            if stdout == slave and stdout != None:
                childout = master
            else:
                childout = proc.stdout

            selectlist = []
            outlist = []
            if childin != None:
                selectlist.append(sys.stdin)
            if childout != None:
                selectlist.append(childout)
                outlist.append(childout)

            while proc.poll() == None or select.select(outlist, [], [], 0)[0]:
                    ready = select.select(selectlist, [], [], 0.1)[0]
                    for f in ready:
                        if f == childout:
                            data = os.read(childout.fileno(), 1024)
                            if data == "": # EOF
                                outlist.remove(childout)
                                selectlist.remove(childout)
                                continue
                            if outf: data = outf(data)
                            if data != None:
                                sys.stdout.write(data)
                                sys.stdout.flush()
                        if f == sys.stdin:
                            if proc.poll() != None:
                                # process terminated, does not take input anymore
                                selectlist.remove(sys.stdin)
                                continue
                            data = os.read(sys.stdin.fileno(), 1024)
                            if data == "": # EOF
                                selectlist.remove(sys.stdin)
                                continue
                            if inf: data = inf(data)
                            if data != None:
                                childin.write(data)
                                childin.flush()
            ret = proc.wait()
            if ret:
                error(dirname, "failed command: %s returned %i"%(" ".join(arglist), ret))
        except (OSError, IOError), e:
            error(dirname, "failed command: %s: %s"%(" ".join(arglist),str(e)))

def preparermligs(texname, dirname, tmplist):
    jobname=texname[:-4]
    try:
        texfile = open(os.path.join(dirname, texname))
    except IOError, (errno, strerror):
        error(os.path.join(dirname, texname), "could not be read: %s. Not using rmligs."%strerror)
        return texname
    else:
        try:
            (subdir, texbasename) = os.path.split(texname)
            rmligsbasename = ".%s.rmligs.tex"%texbasename[:-4]
            rmligsname = os.path.join(subdir, rmligsbasename)
            if options.act:
                rmligsfile = open(os.path.join(dirname, rmligsname), "w")
            else:
                rmligsfile = null
        except IOError, (errno, strerror):
            error(os.path.join(dirname, rmligsname), "could not be written: %s. Not using rmligs."%strerror)
            return texname
        else:
            tmplist.append(rmligsname)

            def rewriteinputrmligs(m):
                name=m.group(2)
                if not os.path.isfile(os.path.join(dirname,name)):
                    name+=".tex"
                if os.path.isfile(os.path.join(dirname, name)):
                    name = preparermligs(name, dirname, tmplist)
                else:
                    error(os.path.join(dirnam, texname), "%s: File could not be located. Not using rmligs."%name, warning=True)
                return "\\%s{%s}"%(m.group(1), name[:-4])
            def filteroutput(d):
                rmligsfile.write(re_inputinclude.sub(rewriteinputrmligs, d))
                return None
            run([rmligs_name, "-f"], dirname, inf=texfile, outf=filteroutput)
            
            texfile.close()
            if rmligsfile is not null:
                rmligsfile.close()
            return rmligsname

re_rmligsfile = re.compile("\\.([^/ ]+)\\.rmligs\\.tex")
def runtex(tex, texname, realname, dirname, reason=""):
    if not options.showoutput:
        outf = None
    else:
        def outf(d):
            return re_rmligsfile.sub("\\1.tex", d)
    if texname != realname:
        run([tex, "-interaction", options.interactionmode, "-jobname", texname[:-4], realname], dirname, outf=outf, reason=reason)
    else:
        run([tex, "-interaction", options.interactionmode, texname], dirname, outf=outf, reason=reason)

# main program:
try:
    if args:
        processtexfiles(args)
    else:
        processtexfiles(["."])

    if options.summary in [ "files", "both" ] :
        if procfiles:
            print "Processed the following files:"
            for f in procfiles:
                print "    %s"%f
        else:
            print "Processed no files."
    if options.summary in [ "failures", "both" ] :
        if errors:
            print "The following problems occured:"
            for f in errors:
                print "    %s"%f

    if errors:
        sys.exit(1)

except KeyboardInterrupt:
    sys.exit(2)