path: root/texall

#!/usr/bin/env python

from optparse import OptionParser
import sys
import os
import os.path
import re
import subprocess

parser = OptionParser(usage="%prog [options] DIR/FILE",
                      version="%prog $Id: $",
                      description="Call LaTeX and friends to typeset all tex files in DIR/FILE. By default up-to-date files are skipped. Care is taken to run all required commands until the result is stable. By default, pdflatex is used, but use of pstricks forces dvi->ps->pdf. The utility program rmligs ist used to improve output for german language texts. Use \"texall -Sqfi batchmode file.tex\" to simulate the behaviour of (pdf)latex in non-interactive mode (useful for emacs).")
parser.add_option("-n", "--dry-run",
                  action="store_false", dest="act", default=True,
                  help="do not run any program")
parser.add_option("-f", "--force",
                  action="store_true", dest="force", default=False,
                  help="regenerate up-to-date files")
parser.add_option("-N", "--non-recursive",
                  action="store_false", dest="recurse", default=True,
                  help="disable recursion into subdirectories")
parser.add_option("-s", "--summary", dest="summary",
                  help="summarise failures and processed files at the end of the run (one of failures (default), files, both, no)",
                  choices=("no","files","failures","both"), default="failures")
parser.add_option("-v", "--verbose",
                  action="count", dest="verbosity", default=1,
                  help="explain what is going on")
parser.add_option("-q", "--quiet",
                  action="store_const", dest="verbosity", const=0,
                  help="suppress progress reports")
parser.add_option("-i", "--interaction",
                  dest="interactionmode", choices=("batchmode", "nonstopmode", "scrollmode", "errorstopmode"), default="batchmode",
                  help="control the behaviour of latex (on of batchmode (default), nonstopmode, scrollmode, errorstopmode)")
parser.add_option("-I", "--ignore",
                  action="store", dest="ignorepattern", default=".*vorlage.*|\.\#.*",
                  help="regular expression of filenames to ignore")
parser.add_option("-P", "--required-pattern",
                  action="store", dest="requiredpattern", default='^\\\\documentclass|%%% TeX-master: t',
                  help="regular expression that must match the file content")
parser.add_option("-T", "--preserve-tempfiles",
                  action="store_false", dest="deletetempfiles", default=True,
                  help="preserve temporary files (default: delete)")
parser.add_option("-S", "--single-run",
                  action="store_true", dest="singlerun", default=False,
                  help="stop after the first LaTeX run, no matter if finished")
(options, args) = parser.parse_args()

re_ignore = None
if options.ignorepattern != "":
    re_ignore = re.compile(options.ignorepattern)

re_required = re.compile(options.requiredpattern)

re_texfile = re.compile(".*\.tex$", re.I)

errors = []
procfiles = []

errstr = {False: "ERROR", True: "WARNING"}
def error(file, desc, warning=False):
    msg = "%s: %s: %s"%(errstr[warning], os.path.normpath(file), desc)
    if options.verbosity:
        print msg
    errors.append(msg)

if os.path.exists("/dev/null"):
    null = file("/dev/null", "wb")
else:
    null = file("nul:", "wb")
    
path = os.path.expandvars("$PATH")
if path == "$PATH":
    path = os.path.defpath
path = path.split(os.path.pathsep)
def isinpath(name):
    for p in path:
        if os.path.isfile(os.path.join(p, name)):
            return True
    return False

if isinpath("rmligs"):
    rmligs_name = "rmligs"
elif isinpath("rmligs-german"):
    rmligs_name = "rmligs-german"
else:
    rmligs_name = ""
    error("rmligs", "Command not found. Please install rmligs (or rmligs-german) for optimal results", warning=True)

# iterate over all .tex files that are not ignored by -i or -p
def alltexfiles(args):
    for a in args:
        if os.path.isdir(a):
            for dir, subdirs, files in os.walk(a):
                if not options.recurse:
                    subdirs[:] = []
                for name in files:
                    if re_texfile.match(name):
                        if re_ignore.match(name):
                            if options.verbosity > 2:
                                print "%s: skipped because of ignored name"%os.path.normpath(os.path.join(dir,name))
                            continue
                        for m in texgrep(re_required, dir, name):
                            yield (dir, name)
                            break
                        else:
                            if options.verbosity > 2:
                                print "%s: skipped because no main latex file"%os.path.normpath(os.path.join(dir,name))
        elif os.path.isfile(a):
            if re_texfile.match(a):
                (dirname, filename) = os.path.split(a)
                if dirname == "":
                    dirname = "."
                yield (dirname, filename)
            else:
                error(a, "is no .tex file; skipped")
        elif os.path.isfile(a+".tex"):
            (dirname, filename) = os.path.split(a+".tex")
            if dirname == "":
                dirname = "."
            yield(dirname, filename)
        else:
            error(a, "file not found")

def texgrep(matcher, dirname, filename, recurse=False):
    try:
        f = open(os.path.join(dirname,filename))
        content = f.read()
        for m in matcher.finditer(content):
            yield m
        if recurse:
            for dep in dependencies(dirname, filename, texonly=True):
                for m in texgrep(matcher, "", dep, recurse=False):
                    yield m
    except IOError, (errno, strerror):
        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
    except StopIteration:
        f.close()
    else:
        f.close()

kpsepaths={}
def texpath(dirname,filename,pathtype="tex",progname="latex",extlist=[],allowsystemfiles=False):
    if not kpsepaths.has_key((pathtype,progname)):
        kpse=subprocess.Popen(["kpsepath", "-n", progname, pathtype], stdout=subprocess.PIPE)
        (out,err)=kpse.communicate()
        if kpse.wait() != 0:
            error("kpsepath", "failed to get paths for %s"%pathtype)
        kpsepaths[(pathtype,progname)] = out[:-1].split(":")
    if os.path.isfile(os.path.join(dirname,filename)):
        return os.path.join(dirname,filename)
    for ext in extlist:
        if os.path.isfile(os.path.join(dirname,filename+ext)):
            return os.path.join(dirname,filename+ext)

RECURSIONDEPTH=10
re_inputinclude = re.compile('\\\\(input|include|@input)\\{([^}]*)\\}')
re_usepackage = re.compile('\\\\usepackage(\\[.*?\\])?\\{([^}]*)\\}')
re_graphics = re.compile('\\\\includegraphics(\\[.*?\\])?\\{([^}]*)\\}')
graphics_ext = ["", ".pdf", ".eps", ".png", ".jpg"]
re_bibliography = re.compile('\\\\bibliography\\{([^}]*)\\}')
re_commawhitespace = re.compile('\\s*,\\s*')
def dependencies(dirname, filename, texonly=False, recurse=RECURSIONDEPTH):
    if not recurse:
        return
    try:
        f = open(os.path.join(dirname,filename))
        content = f.read()
        for m in re_inputinclude.finditer(content):
            texs = m.group(2)
            for tex in re_commawhitespace.split(texs):
                if not os.path.isfile(os.path.join(dirname,tex)):
                    tex+=".tex"
                if os.path.isfile(os.path.join(dirname,tex)):
                    yield os.path.normpath(os.path.join(dirname,tex))
                    for m in dependencies(dirname, tex, texonly, recurse-1):
                        yield m
        for m in re_usepackage.finditer(content):
            stys = m.group(2)
            for sty in re_commawhitespace.split(stys):
                if not os.path.isfile(os.path.join(dirname,sty)):
                    sty+=".sty"
                if os.path.isfile(os.path.join(dirname,sty)):
                    yield os.path.normpath(os.path.join(dirname,sty))
                    for m in dependencies(dirname, sty, texonly, recurse-1):
                        yield m
        if not texonly:
            for m in re_graphics.finditer(content):
                names = m.group(2)
                for name in re_commawhitespace.split(names):
                    for ext in graphics_ext:
                        if os.path.isfile(os.path.join(dirname,name+ext)):
                            yield os.path.normpath(os.path.join(dirname,name+ext))
            for m in re_bibliography.finditer(content):
                bibs = m.group(1)
                for bib in re_commawhitespace.split(bibs):
                    if bib[-4:] != ".bib":
                        bib += ".bib"
                    if bib[-8:] != "-blx.bib":
                        yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib)))
                    
    except IOError, (errno, strerror):
        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
    except StopIteration:
        f.close()
    else:
        f.close()

def bibfiles(dirname, texname):
    for m in texgrep(re_bibliography, dirname, texname, recurse=True):
        bibs = m.group(1)
        for bib in re_commawhitespace.split(bibs):
            if bib[-4:] != ".bib":
                bib += ".bib"
            if bib[-8:] != "-blx.bib":
                yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib)))

def outdatedtexfiles(args):
    for dirname, texname in alltexfiles(args):
        #strip .tex extension
        jobname=texname[:-4]
        pdfname=jobname+".pdf"
        reason = ""
        # check if an update is needed:
        if options.force and not options.verbosity:
            yield (dirname, texname, reason)
        elif not os.path.isfile(os.path.join(dirname,pdfname)):
            if options.verbosity:
                reason = " (because .pdf does not exist)"
            yield (dirname, texname, reason)
        else:
            pdftime = os.path.getmtime(os.path.join(dirname, pdfname))
            if pdftime < os.path.getmtime(os.path.join(dirname, texname)):
                if options.verbosity:
                    reason = " (because .tex is newer than .pdf)"
                yield (dirname, texname, reason)
            else:
                for f in dependencies(dirname, texname):
                    if not os.path.isfile(f):
                        error(os.path.join(dirname,texname), "dependency not found: %s"%f, warning=True)
                        continue
                    if pdftime < os.path.getmtime(f):
                        if options.verbosity:
                            reason = " (because %s is newer than .pdf)"%f
                        yield (dirname, texname, reason)
                        break
                else:
                    r = haderrors(dirname, jobname)
                    if r:
                        if options.verbosity:
                            reason = r
                        yield (dirname, texname, reason)
                    elif options.force:
                        reason = " (because of --force)"
                        yield (dirname, texname, reason)
                    elif options.verbosity > 2:
                        print "%s: skipped because up-to-date"%os.path.normpath(os.path.join(dirname, texname))

re_nopdftex = re.compile('\\\\usepackage(\\[.*?\\])?\\{[^}]*pstricks[^}]*\\}')
def detecttextype(dirname, texname):
    for m in texgrep(re_nopdftex, dirname, texname, recurse=True):
        return "latex"
    return "pdflatex"

re_rmligs = re.compile('\\\usepackage((\\[.*?\\])?\\{n?german\\}|\\[[^]]*?german[^]]*?\\]\\{babel\\})')
def detectrmligs(dirname, texname):
    if rmligs_name == "":
        return False
    for m in texgrep(re_rmligs, dirname, texname, recurse=True):
        return True
    return False

MAXRUNS=5
def processtexfiles(args):
    for dirname, texname, reason in outdatedtexfiles(args):
        procfiles.append(os.path.normpath(os.path.join(dirname,texname)))
        if options.verbosity:
            print "processing %s%s..."%(os.path.normpath(os.path.join(dirname,texname)),reason)

        # list of temporary files to be removed later
        tmplist = []        

        # support for pstricks/plain latex:
        tex = detecttextype(dirname, texname)

        # detect usage of german/ngerman and generate temporary files with rmlig
        realname = texname
        if detectrmligs(dirname, texname):
            realname = preparermligs(texname, dirname, tmplist)
        
        runtex(tex, texname, realname, dirname)

        if not options.singlerun:
            #strip .tex extension
            jobname=texname[:-4]

            # run bibtex if any bibfile changed:
            for bib in bibfiles(dirname, texname):
                bbl = os.path.normpath(os.path.join(dirname, jobname+".bbl"))
                if options.force or not os.path.isfile(bbl)  or os.path.getmtime(bbl) < os.path.getmtime(bib):
                    reason = ""
                    if options.verbosity:
                        if not os.path.isfile(bbl):
                            reason = " (because .bbl does not exist yet)"
                        elif os.path.getmtime(bbl) < os.path.getmtime(bib):
                            reason = " (because %s is newer than .bbl)"%bib
                        else:
                            reason = " (because of --force)"
                    run(["bibtex8", "--wolfgang", jobname], dirname, reason=reason)
                    runtex(tex, texname, realname, dirname, reason=" (because of updated .bbl)")
                    break

            # check for undefined references and run requests
            numrun=0
            reqs=options.act
            while reqs:
                reqs = requests(os.path.join(dirname, jobname+".log"))
                pri = reqs.values()
                pri.sort(reverse=True)
                for p in pri:
                    for req in reqs.keys():
                        rp = reqs[req]
                        if rp == p:
                            reason = ""
                            if options.verbosity:
                                reason = " (because of request in .aux file, priority %d)"%rp
                            if req == "latex":
                                runtex(tex, texname, realname, dirname, reason=reason)
                            elif req == "bibtex":
                                run(["bibtex8", "--wolfgang", jobname], dirname, reason=reason)
                            else:
                                error(os.path.join(dirname,jobname+".aux"), "unsupported request: %s"%req)
                numrun+=1
                if numrun==MAXRUNS:
                    error(os.path.join(dirname, texname), "does not stabilise after %i runs"%MAXRUNS)
                    break

            # update index if it exists - no way of knowing if it was updated
            idx = jobname + ".idx"
            if os.path.isfile(os.path.join(dirname, idx)):
                run(["makeindex", jobname], dirname, reason=" (because .idx file might have changed)")
                runtex(tex, texname, realname, dirname, reason=" (because .ind file might have changed)")

        if tex == "latex":
            run(["dvips", jobname+".dvi"], dirname)
            run(["ps2pdf", jobname+".ps"], dirname)

        rmtempfile(tmplist, dirname)

re_logmatcher = re.compile("^LaTeX Warning: There were undefined references\.$|^LaTeX Warning: Label\(s\) may have changed\. Rerun to get cross-references right\.$|^REQ:(\d+):(\w+):")
def requests(logpath):
    reqs = {}
    def addrequest(name, priority):
        if reqs.has_key(name):
            reqs[name] = max(reqs[name], priority)
        else:
            reqs[name] = priority
    try:
        log = open(logpath)
        for line in log:
            m = re_logmatcher.match(line)
            if m:
                if m.group() == 'LaTeX Warning: There were undefined references.':
                    addrequest("latex",0)
                elif m.group() == 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.':
                    addrequest("latex",0)
                elif m.group(1)!=None:
                    addrequest(m.group(2), int(m.group(1)))
                # TODO: add support for multiple bibliographies, see biblatex.pdf Sec. 2.4.4
    except IOError, (errno, strerror):
        error(logpath, "could not be read: %s"%strerror)
    else:
        log.close()
    return reqs

re_error = re.compile('^! ')
def haderrors(dirname, jobname):
    logpath = os.path.join(dirname, jobname+".log")
    if not os.path.isfile(logpath):
        return None
    try:
        f = open(logpath)
        for line in f:
            if re_error.match(line):
                return " (because of error in .log file)"
            if re_logmatcher.match(line):
                return " (because of request in .log file)"
    except IOError, (errno, strerror):
        error(logpath, "could not be read: %s"%strerror)
    else:
        f.close()
    return False

def rmtempfile(filename, dirname):
    if options.deletetempfiles:
        if not isinstance(filename, list):
            filename = [filename]
        if options.verbosity > 1:
            print "  removing %s"%(", ".join(filename))
        if options.act:
            for f in filename:
                fullname = os.path.join(dirname, f)
                if os.path.isfile(fullname):
                    os.remove(fullname)
                else:
                    error(fullname, "could not be removed (does not exist)", warning=True)

def run(arglist, dirname, reason="", stdin=None, stdout=None, stderr=None, doio=None):
    if options.interactionmode == "batchmode":
        if not stdout:
            stdout=null
        if not stderr:
            stderr=null
    if options.verbosity:
        redir = ""
        if hasattr(stdin, "name"):
            redir = "%s < %s"%(redir, stdin.name)
        if hasattr(stdout, "name") and stdout is not null:
            redir = "%s > %s"%(redir, stdout.name)
        if hasattr(stderr, "name") and stderr is not null:
            redir = "%s 2> %s"%(redir, stderr.name)
        print "  running %s%s%s..."%(" ".join(arglist), redir, reason)
    if options.act:
        proc = subprocess.Popen(arglist, stdin=stdin, stdout=stdout, stderr=stderr, cwd=dirname)
        if doio:
            doio(proc)
        ret = proc.wait()
        if ret:
            error(dirname, "failed command: %s"%(" ".join(arglist)))

def preparermligs(texname, dirname, tmplist):
    jobname=texname[:-4]
    try:
        texfile = open(os.path.join(dirname, texname))
    except IOError, (errno, strerror):
        error(os.path.join(dirname, texname), "could not be read: %s. Not using rmligs."%strerror)
        return texname
    else:
        try:
            (subdir, texbasename) = os.path.split(texname)
            rmligsbasename = ".%s.rmligs.tex"%texbasename[:-4]
            rmligsname = os.path.join(subdir, rmligsbasename)
            if options.act:
                rmligsfile = open(os.path.join(dirname, rmligsname), "w")
            else:
                rmligsfile = null
        except IOError, (errno, strerror):
            error(os.path.join(dirname, rmligsname), "could not be written: %s. Not using rmligs."%strerror)
            return texname
        else:
            tmplist.append(rmligsname)

            def rewriteinputrmligs(m):
                name=m.group(2)
                if not os.path.isfile(os.path.join(dirname,name)):
                    name+=".tex"
                if os.path.isfile(os.path.join(dirname, name)):
                    name = preparermligs(name, dirname, tmplist)
                else:
                    error(os.path.join(dirnam, texname), "%s: File could not be located. Not using rmligs."%name, warning=True)
                return "\\%s{%s}"%(m.group(1), name[:-4])
            def doio(p):
                for line in p.stdout:
                    rmligsfile.write(re_inputinclude.sub(rewriteinputrmligs, line))
            run([rmligs_name, "-f"], dirname, stdin=texfile, stdout=subprocess.PIPE, doio=doio)
            
            texfile.close()
            if rmligsfile is not null:
                rmligsfile.close()
            return rmligsname

re_rmligsfile = re.compile("\\.([^/ ]+)\\.rmligs\\.tex")
def rmligscleanoutput(p): # filter tex output to make emacs jump to the right source positions
    for l in p.stdout:
        sys.stdout.write(re_rmligsfile.sub("\\1.tex", l))
        sys.stdout.flush()
def runtex(tex, texname, realname, dirname, reason=""):
    if texname != realname:
        if options.interactionmode == "batchmode":
            stdout = None
            doio = None
        else:
            stdout = subprocess.PIPE
            doio = rmligscleanoutput
        run([tex, "-interaction", options.interactionmode, "-jobname", texname[:-4], realname], dirname, stdout=stdout, doio=doio, reason=reason)
    else:
        run([tex, "-interaction", options.interactionmode, texname], dirname, reason=reason)

# main program:
try:
    if args:
        processtexfiles(args)
    else:
        processtexfiles(["."])

    if options.summary in [ "files", "both" ] :
        if procfiles:
            print "Processed the following files:"
            for f in procfiles:
                print "    %s"%f
        else:
            print "Processed no files."
    if options.summary in [ "failures", "both" ] :
        if errors:
            print "The following problems occured:"
            for f in errors:
                print "    %s"%f

    if errors:
        sys.exit(1)

except KeyboardInterrupt:
    sys.exit(2)