texall script: do not repeatedly open the same file, use kpsewhich

1 files changed, 579 insertions, 405 deletions

diff --git a/texall b/texall
index afc2b37..6a9f4d7 100755
--- a/texall
+++ b/texall
@@ -1,5 +1,8 @@
 #!/usr/bin/env python
 
+from __future__ import with_statement
+
+from contextlib import closing
 from optparse import OptionParser
 import sys
 import os
@@ -10,61 +13,358 @@ import select
 import subprocess
 import fnmatch
 import dircache
+import termios
+import signal
+
+class TexallOptionParser(OptionParser):
+    def __init__(self):
+        OptionParser.__init__(self,
+                              usage="%prog [options] DIR/FILE",
+                              version="%prog $Id$",
+                              description="Call LaTeX and friends to typeset all tex files in DIR/FILE. By default up-to-date files are skipped. Care is taken to run all required commands until the result is stable. By default, pdflatex is used, but use of pstricks forces dvi->ps->pdf. The utility program rmligs ist used to improve output for german language texts. Use \"texall -Sqfi batchmode file.tex\" to simulate the behaviour of (pdf)latex in non-interactive mode (useful for emacs).")
+        self.add_option("-n", "--dry-run",
+                        action="store_false", dest="act", default=True,
+                        help="do not run any program")
+        self.add_option("-f", "--force",
+                        action="store_true", dest="force", default=False,
+                        help="regenerate up-to-date files")
+        self.add_option("-N", "--non-recursive",
+                        action="store_false", dest="recurse", default=True,
+                        help="disable recursion into subdirectories")
+        self.add_option("-s", "--summary", dest="summary",
+                        help="summarise failures and processed files at the end of the run (one of failures (default), files, both, no)",
+                        choices=("no","files","failures","both"), default="failures")
+        self.add_option("-v", "--verbose",
+                        action="count", dest="verbosity", default=1,
+                        help="explain what is going on")
+        self.add_option("-q", "--quiet",
+                        action="store_const", dest="verbosity", const=0,
+                        help="suppress progress reports")
+        self.add_option("-o", "--show-output",
+                        action="store_true", dest="showoutput", default=False,
+                        help="show the output of the called programs")
+        self.add_option("-i", "--interaction",
+                        dest="interactionmode", choices=("batchmode", "nonstopmode", "scrollmode", "errorstopmode"),
+                        help="control the behaviour of latex (on of batchmode (default), nonstopmode (default for -o), scrollmode, errorstopmode)")
+        self.add_option("-I", "--ignore",
+                        action="store", dest="ignorepattern", default=".*vorlage.*|\.\#.*|\..*\.rmligs\.tex",
+                        help="regular expression of filenames to ignore")
+        self.add_option("-P", "--required-pattern",
+                        action="store", dest="requiredpattern", default='^\\\\documentclass|%%% TeX-master: t',
+                        help="regular expression that must match the file content")
+        self.add_option("-T", "--preserve-tempfiles",
+                        action="store_false", dest="deletetempfiles", default=True,
+                        help="preserve temporary files (default: delete)")
+        self.add_option("-K", "--no-kpathsea",
+                        action="store_true", dest="nokpse", default=False,
+                        help="do not call kpsewhich (might be faster)")
+        self.add_option("-S", "--single-run",
+                        action="store_true", dest="singlerun", default=False,
+                        help="stop after the first LaTeX run, no matter if finished")
+
+    def parse_args(self, args=None, values=None):
+        options, arguments = OptionParser.parse_args(self, args, values)
+
+        # provide default for interactionmode:
+        if options.interactionmode == None:
+            if options.showoutput:
+                options.interactionmode = "nonstopmode"
+            else:
+                options.interactionmode = "batchmode"
+        
+        return options, arguments
 
-optparse = OptionParser(usage="%prog [options] DIR/FILE",
-                        version="%prog $Id: $",
-                        description="Call LaTeX and friends to typeset all tex files in DIR/FILE. By default up-to-date files are skipped. Care is taken to run all required commands until the result is stable. By default, pdflatex is used, but use of pstricks forces dvi->ps->pdf. The utility program rmligs ist used to improve output for german language texts. Use \"texall -Sqfi batchmode file.tex\" to simulate the behaviour of (pdf)latex in non-interactive mode (useful for emacs).")
-optparse.add_option("-n", "--dry-run",
-                    action="store_false", dest="act", default=True,
-                    help="do not run any program")
-optparse.add_option("-f", "--force",
-                    action="store_true", dest="force", default=False,
-                    help="regenerate up-to-date files")
-optparse.add_option("-N", "--non-recursive",
-                    action="store_false", dest="recurse", default=True,
-                    help="disable recursion into subdirectories")
-optparse.add_option("-s", "--summary", dest="summary",
-                    help="summarise failures and processed files at the end of the run (one of failures (default), files, both, no)",
-                    choices=("no","files","failures","both"), default="failures")
-optparse.add_option("-v", "--verbose",
-                    action="count", dest="verbosity", default=1,
-                    help="explain what is going on")
-optparse.add_option("-q", "--quiet",
-                    action="store_const", dest="verbosity", const=0,
-                    help="suppress progress reports")
-optparse.add_option("-o", "--show-output",
-                    action="store_true", dest="showoutput", default=False,
-                    help="show the output of the called programs")
-optparse.add_option("-i", "--interaction",
-                    dest="interactionmode", choices=("batchmode", "nonstopmode", "scrollmode", "errorstopmode"),
-                    help="control the behaviour of latex (on of batchmode (default), nonstopmode (default for -o), scrollmode, errorstopmode)")
-optparse.add_option("-I", "--ignore",
-                    action="store", dest="ignorepattern", default=".*vorlage.*|\.\#.*|\..*\.rmligs\.tex",
-                    help="regular expression of filenames to ignore")
-optparse.add_option("-P", "--required-pattern",
-                    action="store", dest="requiredpattern", default='^\\\\documentclass|%%% TeX-master: t',
-                    help="regular expression that must match the file content")
-optparse.add_option("-T", "--preserve-tempfiles",
-                    action="store_false", dest="deletetempfiles", default=True,
-                    help="preserve temporary files (default: delete)")
-optparse.add_option("-S", "--single-run",
-                    action="store_true", dest="singlerun", default=False,
-                    help="stop after the first LaTeX run, no matter if finished")
-(opts, programargs) = optparse.parse_args()
-
-if opts.interactionmode == None:
-    if opts.showoutput:
-        opts.interactionmode = "nonstopmode"
-    else:
-        opts.interactionmode = "batchmode"
+class AnalyserData:
+    def __init__(self, name, filetype):
+        self.name = name
+        self.filetype = filetype
 
-re_ignore = None
-if opts.ignorepattern != "":
-    re_ignore = re.compile(opts.ignorepattern)
+class Analyser():
+    """Abstract class for analysing of program output and file content"""
+    
+    def __init__(self, d):
+        """Instance creation.
 
-re_required = re.compile(opts.requiredpattern)
+        d -- dictionary to store data in"""
+        self.d = d
+        
+    def parse(self, text):
+        """parse text and change own state accordingly.
+
+        text -- the filecontent or line to analyse"""
+        raise NotImplementedError
+
+    def merge(self, d):
+        """merge information gathered from an included file.
+
+        d -- dictionary with information gathered from the included file"""
+        raise NotImplementedError
+
+    def finish(self, dirname):
+        """final steps after all dependecy file information hat been merged.
+
+        Only executed if this data object belongs to a main .tex file.
+
+        dirname -- directory of current job"""
+        pass
+
+class AnalyseTexRequired(Analyser):
+    def __init__(self, d):
+        d.required = False
+        Analyser.__init__(self,d)
+    def parse(self, text):
+        if re_required.match(text):
+            self.d.required = True
+    def merge(self, d):
+        pass # do not take over master file status
+
+class AnalyseTexDependencies(Analyser):
+    def __init__(self, d):
+        d.deps = []
+        d.grfdeps = []
+        Analyser.__init__(self, d)
+    def parse(self, text):
+        for m in re_inputinclude.finditer(text):
+            self.d.deps.append((m.group(2), (".tex", "")))
+        m = re_documentclass.match(text)
+        if m:
+            cls = m.group(2)
+            self.d.deps.append((cls, (".cls", "")))
+        for m in re_usepackage.finditer(text):
+            stys = m.group(2)
+            for sty in re_commawhitespace.split(stys):
+                self.d.deps.append((sty, (".sty", "")))
+        for m in re_graphics.finditer(text):
+            self.d.grfdeps.append(m.group(2))
+    def merge(self, d):
+        self.d.deps.extend(d.deps)
+        self.d.grfdeps.extend(d.grfdeps)
+    def finish(self, _):
+        if hasattr(self.d, "needsPS"):
+            if self.d.needsPS:
+                for f in self.d.grfdeps:
+                    self.d.deps.append((f, (".eps", ".jpg", ".png", "")))
+            else:
+                for f in self.d.grfdeps:
+                    self.d.deps.append((f, (".pdf", ".jpg", ".png", "")))
+
+# TODO: Analyser for \usebeamerXXXtheme
+
+class AnalyseTexBibfiles(Analyser):
+    def __init__(self, d):
+        d.bibfiles = []
+        Analyser.__init__(self, d)
+    def parse(self, text):
+        for m in re_bibliography.finditer(text):
+            bibs = m.group(1)
+            for bib in re_commawhitespace.split(bibs):
+                if bib[-8:] != "-blx.bib" and bib[-4:] != "-blx":
+                    self.d.bibfiles.append(bib)
+    def merge(self, d):
+        self.d.bibfiles.extend(d.bibfiles)
+    def finish(self, dirname):
+        bibpaths = []
+        for bib in self.d.bibfiles:
+            bibpath = texpath(dirname, bib, pathtype="bib",progname="bibtex")
+            bibpath = strippath(bibpath, dirname)
+            self.d.deps.append((bibpath, ("",)))
+            bibpaths.append(bibpath)
+        self.d.bibfiles = bibpaths
+
+class AnalyseTexNeedsPs(Analyser):
+    def __init__(self, d):
+        d.needsPS = False
+        Analyser.__init__(self, d)
+    def parse(self, text):
+        if re_needsps.match(text):
+            self.d.needsPS = True
+    def merge(self, d):
+        self.d.needsPS |= d.needsPS
+
+class AnalyseTexRmligs(Analyser):
+    def __init__(self, d):
+        d.rmligs = False
+        Analyser.__init__(self, d)
+    def parse(self, text):
+        if re_rmligs.match(text):
+            self.d.rmligs = True
+    def merge(self, d):
+        self.d.rmligs |= d.rmligs
+
+texanalysers = (AnalyseTexRequired, AnalyseTexDependencies, AnalyseTexBibfiles, AnalyseTexNeedsPs, AnalyseTexRmligs)
+
+class AnalyseLogErrors(Analyser):
+    def __init__(self, d):
+        d.errors = False
+        Analyser.__init__(self, d)
+    def parse(self, line):
+        if re_error.match(line):
+            self.d.errors = True
+
+class AnalyseLogRequests(Analyser):
+    def __init__(self, d):
+        d.requests = {}
+        Analyser.__init__(self, d)
+        self.inrequest = False
+    def addrequest(self, name, priority):
+        if self.d.requests.has_key(name):
+            self.d.requests[name] = max(self.d.requests[name], priority)
+        else:
+            self.d.requests[name] = priority
+    def parse(self, line):
+        # line by line
+        if self.inrequest:
+            # strip newline character
+            while line[-1] in ("\n", "\r"):
+                line = line[:-1]
+            if line[:7] == "binary=":
+                self.reqbinary = line[7:]
+            elif line[:7] == "option=":
+                self.reqoptions.extend(line[7:].split(" "))
+            elif line[:7] == "infile=":
+                self.reqinfile = line[7:]
+            elif line == ":REQ":
+                if self.reqbinary == None:
+                    self.reqbinary = self.reqname
+                if self.reqbinary not in ("bibtex", "bibtex8"):
+                    error(self.d.name, "ignoring request to run %s for security reasons"%self.reqbinary, warning=True)
+                    self.inrequest = False
+                    return
+                self.reqoptions[:0] = [self.reqbinary]
+                if self.reqbinary == "bibtex8":
+                    self.reqoptions[1:1] = [ "--wolfgang" ]
+                if self.reqinfile != None:
+                    self.reqoptions.append(self.reqinfile)
+                self.addrequest(tuple(self.reqoptions), self.reqpri)
+                self.inrequest=False
+            else:
+                error(self.d.name, "malformatted request ignored", warning=True)
+                self.inrequest=False
+        else:
+            m = re_logmatcher.match(line)
+            if m:
+                if m.group() == 'LaTeX Warning: There were undefined references.':
+                    self.addrequest("latex",0)
+                elif m.group() == 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.':
+                    self.addrequest("latex",0)
+                elif m.group(1)!=None:
+                    self.addrequest(m.group(2), int(m.group(1)))
+                elif m.group(3)!=None:
+                    self.inrequest = True
+                    self.reqpri = int(m.group(3))
+                    self.reqname = m.group(4)
+                    self.reqbinary = None
+                    self.reqoptions = []
+                    self.reqinfile = None
+
+loganalysers = (AnalyseLogErrors, AnalyseLogRequests)
+
+analysecache = {}
+
+def analyseFile(dirname, filename, analyserclasses, parsetype, fulltext=False, cache=False, recurse=None, recursesystemfiles=False):
+    pathname = os.path.join(dirname, filename)
+    if cache and (parsetype, pathname) in analysecache:
+        return analysecache[(parsetype, pathname)]
+    data = AnalyserData(pathname, parsetype)
+    analysers = []
+    for analyser in analyserclasses:
+        analysers.append(analyser(data))
+    with closing(open(os.path.join(dirname, filename), "r")) as f:
+        if fulltext:
+            text = f.read()
+            for a in analysers:
+                a.parse(text)
+        else:
+            for line in f:
+                for a in analysers:
+                    a.parse(line)
+    if hasattr(data, "deps") and (recurse
+                                  or (recurse==None
+                                      and hasattr(data, "requrired")
+                                      and data.required)):
+        # recurse:
+        for dep, extlist in data.deps:
+            deppath = texpath(dirname, dep, extlist=extlist)
+            if (not recursesystemfiles) and re_systemfile.match(deppath): # skip system files
+                continue
+            depname = strippath(deppath, dirname)
+            depdata = analyseFile(dirname, depname, fulltext, cache, recurse=True)
+            for a in analysers:
+                a.merge(depdata)
+    if hasattr(data, "required") and data.required:
+        for a in analysers:
+            a.finish(dirname)
+    if cache:
+        analysecache[(parsetype, pathname)] = data
+    return data
+
+def analyseTex(dirname, filename):
+    return analyseFile(dirname, filename, texanalysers, "tex", fulltext=True, cache=True)
+
+def analyseLog(dirname, filename):
+    return analyseFile(dirname, filename, loganalysers, "log", cache=False)
+
+kpsecache = {}
+kpseproc = {}
+def texpath(dirname,filename,pathtype="tex",progname="latex",extlist=("",)):
+    filename = os.path.expanduser(filename)
+    # first try files in current directory
+    for ext in extlist:
+        if os.path.isfile(os.path.join(dirname,filename+ext)):
+            return filename+ext
+    if opts.nokpse:
+        raise ValueError("file not found, not using kpsewhich: %s (extensions: %r)")
+    # call kpsewhich to locate the file
+    if not kpsecache.has_key((pathtype,progname)):
+        kpsecache[(pathtype,progname)] = {}
+    cache = kpsecache[(pathtype,progname)]
+    for ext in extlist:
+        lookupname = filename + ext
+        if cache.has_key(lookupname):
+            if cache[lookupname]:
+                return cache[lookupname]
+            else:
+                continue
+        else:
+            if iswindows:
+                kpse=subprocess.Popen(["kpsewhich", "-format", pathtype, "-progname", progname, lookupname], stdout=subprocess.PIPE)
+                (out,err)=kpse.communicate()
+                if kpse.wait() == 0:
+                    pathname = out[:-1]
+                    cache[lookupname] = pathname
+                    return pathname
+                else:
+                    cache[lookupname] = None
+            else:
+                if kpseproc.has_key((pathtype, progname)):
+                    _, master = kpseproc[(pathtype, progname)]
+                    os.write(master, lookupname + "\n")
+                else:
+                    master, slave = pty.openpty()
 
-re_texfile = re.compile(".*\.tex$", re.I)
+                    #turn off echo
+                    attr = termios.tcgetattr(slave)
+                    attr[3] = attr[3] & ~termios.ECHO
+                    termios.tcsetattr(slave, termios.TCSANOW, attr)
+                    
+                    proc = subprocess.Popen(["kpsewhich", "-interactive",
+                                             "-format", pathtype,
+                                             "-progname", progname, lookupname],
+                                            stdout=slave,
+                                            stdin=slave)
+                    kpseproc[(pathtype, progname)] = (proc, master)
+                if select.select([master], [], [], 0.1)[0]:
+                    pathname = os.read(master, 1024)
+                else:
+                    pathname = None
+                if pathname:
+                    while pathname[-1] in ("\n", "\r"):
+                        pathname = pathname[:-2]
+                    cache[lookupname] = pathname
+                    return pathname
+                else:
+                    cache[lookupname] = None
+    raise ValueError("file not found by kpsewhich: %s (extensions: %r)"%(filename,extlist))
 
 errors = []
 procfiles = []
@@ -81,11 +381,12 @@ null = file(os.path.devnull, "wb")
 # platform checking:
 iswindows = (sys.platform == "win32")
 
-# provide a copy of os.path.relpath from python 2.6
+# provide a (modified) copy of os.path.relpath from python 2.6
 def relpath(pathname, start=os.path.curdir):
     """Return a relative version of a path"""
-        
+    
     if not iswindows:
+        pathname = os.path.expanduser(pathname)
         start_list = os.path.abspath(start).split(os.path.sep)
         path_list = os.path.abspath(pathname).split(os.path.sep)
     
@@ -116,6 +417,15 @@ def relpath(pathname, start=os.path.curdir):
         rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
         return os.path.join(*rel_list)
 
+def strippath(pathname, start):
+    pathname = os.path.expanduser(pathname)
+    pathname = os.path.abspath(os.path.join(start, pathname))
+    start = os.path.abspath(start)
+    if pathname[:len(start)] == start:
+        return pathname[len(start)+1:]
+    else:
+        return pathname
+
 binpath = os.path.expandvars("$PATH")
 if binpath == "$PATH":
     binpath = os.path.defpath
@@ -134,8 +444,8 @@ else:
     rmligs_name = ""
     error("rmligs", "Command not found. Please install rmligs (or rmligs-german) for optimal results", warning=True)
 
-# iterate over all .tex files that are not ignored by -i or -p
 def alltexfiles(arglist):
+    """iterate over all .tex files that are not ignored by -I"""
     for a in arglist:
         if os.path.isdir(a):
             for directory, subdirs, files in os.walk(a):
@@ -146,13 +456,8 @@ def alltexfiles(arglist):
                         if re_ignore.match(name):
                             if opts.verbosity > 2:
                                 print "%s: skipped because of ignored name"%os.path.normpath(os.path.join(directory,name))
-                            continue
-                        for _ in texgrep(re_required, directory, name):
-                            yield (directory, name)
-                            break
                         else:
-                            if opts.verbosity > 2:
-                                print "%s: skipped because no main latex file"%os.path.normpath(os.path.join(directory,name))
+                            yield (directory, name)
         elif os.path.isfile(a):
             if re_texfile.match(a):
                 (dirname, filename) = os.path.split(a)
@@ -169,325 +474,154 @@ def alltexfiles(arglist):
         else:
             error(a, "file not found")
 
-def texgrep(matcher, dirname, filename, recurse=False):
-    try:
-        f = open(os.path.join(dirname,filename))
-        content = f.read()
-        for m in matcher.finditer(content):
-            yield m
-        if recurse:
-            for dep in dependencies(dirname, filename, texonly=True):
-                for m in texgrep(matcher, "", dep, recurse=False):
-                    yield m
-    except IOError, (errno, strerror):
-        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
-    except StopIteration:
-        f.close()
-    else:
-        f.close()
-
-kpsepaths={}
-#TODO: finish implementation, use in search
-def texpath(dirname,filename,pathtype="tex",progname="latex",extlist=[],allowsystemfiles=False):
-    if not kpsepaths.has_key((pathtype,progname)):
-        kpse=subprocess.Popen(["kpsepath", "-n", progname, pathtype], stdout=subprocess.PIPE)
-        (out,err)=kpse.communicate()
-        if kpse.wait() != 0:
-            error("kpsepath", "failed to get paths for %s"%pathtype)
-        kpsepaths[(pathtype,progname)] = out[:-1].split(":")
-    if os.path.isfile(os.path.join(dirname,filename)):
-        return os.path.join(dirname,filename)
-    for ext in extlist:
-        if os.path.isfile(os.path.join(dirname,filename+ext)):
-            return os.path.join(dirname,filename+ext)
-
-RECURSIONDEPTH=10
-re_inputinclude = re.compile('\\\\(input|include|@input)\\{([^}]*)\\}')
-re_usepackage = re.compile('\\\\usepackage(\\[.*?\\])?\\{([^}]*)\\}')
-re_graphics = re.compile('\\\\includegraphics(\\[.*?\\])?\\{([^}]*)\\}')
-graphics_ext = ["", ".pdf", ".eps", ".png", ".jpg"]
-re_bibliography = re.compile('\\\\bibliography\\{([^}]*)\\}')
-re_commawhitespace = re.compile('\\s*,\\s*')
-def dependencies(dirname, filename, texonly=False, recurse=RECURSIONDEPTH):
-    if not recurse:
-        return
-    try:
-        f = open(os.path.join(dirname,filename))
-        content = f.read()
-        for m in re_inputinclude.finditer(content):
-            texs = m.group(2)
-            for tex in re_commawhitespace.split(texs):
-                if not os.path.isfile(os.path.join(dirname,tex)):
-                    tex+=".tex"
-                if os.path.isfile(os.path.join(dirname,tex)):
-                    yield os.path.normpath(os.path.join(dirname,tex))
-                    for m in dependencies(dirname, tex, texonly, recurse-1):
-                        yield m
-        for m in re_usepackage.finditer(content):
-            stys = m.group(2)
-            for sty in re_commawhitespace.split(stys):
-                if not os.path.isfile(os.path.join(dirname,sty)):
-                    sty+=".sty"
-                if os.path.isfile(os.path.join(dirname,sty)):
-                    yield os.path.normpath(os.path.join(dirname,sty))
-                    for m in dependencies(dirname, sty, texonly, recurse-1):
-                        yield m
-        if not texonly:
-            for m in re_graphics.finditer(content):
-                names = m.group(2)
-                for name in re_commawhitespace.split(names):
-                    for ext in graphics_ext:
-                        if os.path.isfile(os.path.join(dirname,name+ext)):
-                            yield os.path.normpath(os.path.join(dirname,name+ext))
-            for m in re_bibliography.finditer(content):
-                bibs = m.group(1)
-                for bib in re_commawhitespace.split(bibs):
-                    if bib[-4:] != ".bib":
-                        bib += ".bib"
-                    if bib[-8:] != "-blx.bib":
-                        yield os.path.normpath(os.path.join(dirname,os.path.expanduser(bib)))
-                    
-    except IOError, (errno, strerror):
-        error(os.path.join(dirname, filename), "could not be read: %s"%strerror)
-    except StopIteration:
-        f.close()
-    else:
-        f.close()
-
-def bibfiles(dirname, texname):
-    for m in texgrep(re_bibliography, dirname, texname, recurse=True):
-        bibs = m.group(1)
-        for bib in re_commawhitespace.split(bibs):
-            if bib[-4:] != ".bib":
-                bib += ".bib"
-            if bib[-8:] != "-blx.bib":
-                yield bib
-
-def outdatedtexfiles(arglist):
-    for dirname, texname in alltexfiles(arglist):
-        #strip .tex extension
-        jobname=texname[:-4]
-        pdfname=jobname+".pdf"
-        reason = ""
-        # check if an update is needed:
-        if opts.force and not opts.verbosity:
-            yield (dirname, texname, reason)
-        elif not os.path.isfile(os.path.join(dirname,pdfname)):
-            if opts.verbosity:
-                reason = " (because .pdf does not exist)"
-            yield (dirname, texname, reason)
-        else:
-            pdftime = os.path.getmtime(os.path.join(dirname, pdfname))
-            if pdftime < os.path.getmtime(os.path.join(dirname, texname)):
-                if opts.verbosity:
-                    reason = " (because .tex is newer than .pdf)"
-                yield (dirname, texname, reason)
-            else:
-                for f in dependencies(dirname, texname):
-                    if not os.path.isfile(f):
-                        error(os.path.join(dirname,texname), "dependency not found: %s"%f, warning=True)
-                        continue
-                    if pdftime < os.path.getmtime(f):
-                        if opts.verbosity:
-                            reason = " (because %s is newer than .pdf)"%f
-                        yield (dirname, texname, reason)
-                        break
-                else:
-                    r = haderrors(dirname, jobname)
-                    if r:
-                        if opts.verbosity:
-                            reason = r
-                        yield (dirname, texname, reason)
-                    elif opts.force:
-                        reason = " (because of --force)"
-                        yield (dirname, texname, reason)
-                    elif opts.verbosity > 2:
-                        print "%s: skipped because up-to-date"%os.path.normpath(os.path.join(dirname, texname))
-
-re_nopdftex = re.compile('\\\\usepackage(\\[.*?\\])?\\{[^}]*pstricks[^}]*\\}')
-def detecttextype(dirname, texname):
-    for _ in texgrep(re_nopdftex, dirname, texname, recurse=True):
-        return "latex"
-    return "pdflatex"
-
-re_rmligs = re.compile('\\\usepackage((\\[.*?\\])?\\{n?german\\}|\\[[^]]*?german[^]]*?\\]\\{babel\\})')
-def detectrmligs(dirname, texname):
-    if rmligs_name == "":
+def needsupdate(dirname, texname, data=None):
+    jobname = texname[:-4]
+
+    if data == None:
+        data = analyseTex(dirname, texname)
+
+    # skip non-main files
+    if not data.required:
+        if opts.verbosity > 2:
+            print "%s: skipped because no main latex file"%os.path.normpath(os.path.join(dirname,texname))
         return False
-    for _ in texgrep(re_rmligs, dirname, texname, recurse=True):
+
+    # silent force
+    if opts.force and not opts.verbosity:
         return True
+
+    # check for nonexistant output file
+    outname = jobname + ".pdf"
+    if not os.path.isfile(os.path.join(dirname, outname)):
+        return " (because .pdf does not exist)"
+    
+    # check for updated source and dependency files
+    outdate = os.path.getmtime(os.path.join(dirname, outname))
+    if outdate < os.path.getmtime(os.path.join(dirname, texname)):
+        return " (because .tex is newer than .pdf)"
+    for filename, extlist in data.deps:
+        try:
+            deppath = texpath(dirname, filename, extlist=extlist)
+            if outdate < os.path.getmtime(os.path.join(dirname, deppath)):
+                return " (because %s is newer than .pdf)"%deppath
+        except ValueError, e:
+            if opts.verbosity > 2:
+                error(os.path.join(dirname, texname), "ignored dependency: " + str(e), warning=True)
+
+    # check for errors and requests
+    logdata = analyseLog(dirname, jobname + ".log")
+    if logdata.errors:
+        return " (because of error(s) in .log file)"
+    if logdata.requests:
+        return " (because of request(s) in .log file)"
+    
+    # verbose force
+    if opts.force:
+        return " (because of --force)"
+
+    # up to date
+    if opts.verbosity > 2:
+        print "%s: skipped because up-to-date"%os.path.normpath(os.path.join(dirname, texname))
     return False
+        
+MAXRUNS=5
+def processTex(dirname, texname, data=None, reason=""):
+    procfiles.append(os.path.normpath(os.path.join(dirname,texname)))
+
+    if data == None:
+        data = analyseTex(dirname, texname)
+    
+    if opts.verbosity:
+        print "processing %s%s..."%(os.path.normpath(os.path.join(dirname,texname)),reason)
+
+    # list of temporary files to be removed later
+    tmplist = []        
+
+    # support for pstricks/plain latex:
+    if data.needsPS:
+        tex = "latex"
+    else:
+        tex = "pdflatex"
+
+    # detect usage of german/ngerman and generate temporary files with rmlig
+    realname = texname
+
+    if data.rmligs:
+        realname = preparermligs(texname, dirname, tmplist)
+
+    #strip .tex extension
+    jobname=texname[:-4]
+
+    # remove all outdated .bbl files (will be recreated by requests)
+    bbls = bblfiles(dirname, jobname)
+    for bib in data.bibfiles:
+        bibpath = os.path.join(dirname, bib)
+        bibtime = os.path.getmtime(bibpath)
+        nextbbls = bbls[:]
+        for bbl in bbls:
+            bblpath = os.path.normpath(os.path.join(dirname, bbl))
+            bbltime = os.path.getmtime(bblpath)
+            if (opts.force and not opts.singlerun) or bbltime < bibtime:
+                reason = ""
+                if opts.verbosity:
+                    if bbltime < bibtime:
+                        reason = " (because %s is newer than %s)"%(bib,bbl)
+                    else:
+                        reason = " (because of --force)"
+                rmtempfile(bbl, dirname, force=True, reason=reason)
+                nextbbls.remove(bbl)
+        bbls = nextbbls
+
+    runtex(tex, texname, realname, dirname)
+
+    if not opts.singlerun:
+        # check for undefined references and run requests
+        numrun=0
+        reqs=opts.act # ignore requests if programs are not run
+        while reqs:
+            reqs = analyseLog(dirname, jobname+".log").requests
+            if not reqs: break
+            pri = reqs.values()
+            pri.sort(reverse=True)
+            lastp = pri[0] + 1
+            for p in pri:
+                if p == lastp: continue
+                lastp = p
+                for req in reqs.keys():
+                    rp = reqs[req]
+                    if rp == p:
+                        reason = ""
+                        if opts.verbosity:
+                            reason = " (because of request in .aux file, priority %d)"%rp
+                        if req == "latex":
+                            runtex(tex, texname, realname, dirname, reason=reason)
+                        elif isinstance(req, tuple):
+                            run(req, dirname, reason)
+                        else:
+                            error(os.path.join(dirname,jobname+".aux"), "unsupported request: %s"%req)
+            numrun+=1
+            if numrun==MAXRUNS:
+                error(os.path.join(dirname, texname), "does not stabilise after %i runs"%MAXRUNS)
+                break
+
+        # update index if it exists - no way of knowing if it was updated
+        idx = jobname + ".idx"
+        if os.path.isfile(os.path.join(dirname, idx)):
+            run(["makeindex", jobname], dirname, reason=" (because .idx file might have changed)")
+            runtex(tex, texname, realname, dirname, reason=" (because .ind file might have changed)")
+
+    if data.needsPS:
+        run(["dvips", jobname+".dvi"], dirname)
+        run(["ps2pdf", jobname+".ps"], dirname)
+
+    rmtempfile(tmplist, dirname)
 
 def bblfiles(dirname, jobname):
     files = dircache.listdir(dirname)
     return fnmatch.filter(files, "%s*.bbl"%jobname)
 
-MAXRUNS=5
-def processtexfiles(arglist):
-    for dirname, texname, reason in outdatedtexfiles(arglist):
-        procfiles.append(os.path.normpath(os.path.join(dirname,texname)))
-        if opts.verbosity:
-            print "processing %s%s..."%(os.path.normpath(os.path.join(dirname,texname)),reason)
-
-        # list of temporary files to be removed later
-        tmplist = []        
-
-        # support for pstricks/plain latex:
-        tex = detecttextype(dirname, texname)
-
-        # detect usage of german/ngerman and generate temporary files with rmlig
-        realname = texname
-        if detectrmligs(dirname, texname):
-            realname = preparermligs(texname, dirname, tmplist)
-
-        #strip .tex extension
-        jobname=texname[:-4]
-
-        # remove all outdated .bbl files (will be recreated by requests)
-        bbls = bblfiles(dirname, jobname)
-        for bib in bibfiles(dirname, texname):
-            bibpath = os.path.join(dirname, os.path.expanduser(bib))
-            bibtime = os.path.getmtime(bibpath)
-            nextbbls = bbls[:]
-            for bbl in bbls:
-                bblpath = os.path.normpath(os.path.join(dirname, bbl))
-                bbltime = os.path.getmtime(bblpath)
-                if (opts.force and not opts.singlerun) or bbltime < bibtime:
-                    reason = ""
-                    if opts.verbosity:
-                        if bbltime < bibtime:
-                            reason = " (because %s is newer than %s)"%(bib,bbl)
-                        else:
-                            reason = " (because of --force)"
-                    rmtempfile(bbl, dirname, force=True, reason=reason)
-                    nextbbls.remove(bbl)
-            bbls = nextbbls
-
-        runtex(tex, texname, realname, dirname)
-
-        if not opts.singlerun:
-            # check for undefined references and run requests
-            numrun=0
-            reqs=opts.act
-            while reqs:
-                reqs = requests(os.path.join(dirname, jobname+".log"))
-                if not reqs: break
-                pri = reqs.values()
-                pri.sort(reverse=True)
-                lastp = pri[0] + 1
-                for p in pri:
-                    if p == lastp: continue
-                    lastp = p
-                    for req in reqs.keys():
-                        rp = reqs[req]
-                        if rp == p:
-                            reason = ""
-                            if opts.verbosity:
-                                reason = " (because of request in .aux file, priority %d)"%rp
-                            if req == "latex":
-                                runtex(tex, texname, realname, dirname, reason=reason)
-                            elif isinstance(req, tuple):
-                                run(req, dirname, reason)
-                            else:
-                                error(os.path.join(dirname,jobname+".aux"), "unsupported request: %s"%req)
-                numrun+=1
-                if numrun==MAXRUNS:
-                    error(os.path.join(dirname, texname), "does not stabilise after %i runs"%MAXRUNS)
-                    break
-
-            # update index if it exists - no way of knowing if it was updated
-            idx = jobname + ".idx"
-            if os.path.isfile(os.path.join(dirname, idx)):
-                run(["makeindex", jobname], dirname, reason=" (because .idx file might have changed)")
-                runtex(tex, texname, realname, dirname, reason=" (because .ind file might have changed)")
-
-        if tex == "latex":
-            run(["dvips", jobname+".dvi"], dirname)
-            run(["ps2pdf", jobname+".ps"], dirname)
-
-        rmtempfile(tmplist, dirname)
-
-re_logmatcher = re.compile("^LaTeX Warning: There were undefined references\.$|^LaTeX Warning: Label\(s\) may have changed\. Rerun to get cross-references right\.$|^REQ:(\d+):(\w+):REQ|^REQ:(\d+):(\w+):$")
-def requests(logpath):
-    reqs = {}
-    def addrequest(name, priority):
-        if reqs.has_key(name):
-            reqs[name] = max(reqs[name], priority)
-        else:
-            reqs[name] = priority
-    try:
-        log = open(logpath)
-        inrequest = False
-        reqpri = None
-        reqoptions = []
-        for line in log:
-            if inrequest:
-                # strip newline character
-                while line[-1] in ("\n", "\r"):
-                    line = line[:-1]
-                if line[:7] == "binary=":
-                    reqbinary = line[7:]
-                elif line[:7] == "option=":
-                    reqoptions.extend(line[7:].split(" "))
-                elif line[:7] == "infile=":
-                    reqinfile = line[7:]
-                elif line == ":REQ":
-                    if reqbinary != None:
-                        if reqbinary not in ("bibtex", "bibtex8"):
-                            error(logpath, "ignoring request to run %s for security reasons"%reqbinary, warning=True)
-                            inrequest = False
-                            continue
-                        reqoptions[:0] = [reqbinary]
-                        if reqbinary == "bibtex8":
-                            reqoptions[1:1] = [ "--wolfgang" ]
-                    if reqinfile != None:
-                        reqoptions.append(reqinfile)
-                    addrequest(tuple(reqoptions), reqpri)
-                    inrequest=False
-                else:
-                    error(logpath, "malformatted request ignored", warning=True)
-                    inrequest=False
-            else:
-                m = re_logmatcher.match(line)
-                if m:
-                    if m.group() == 'LaTeX Warning: There were undefined references.':
-                        addrequest("latex",0)
-                    elif m.group() == 'LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right.':
-                        addrequest("latex",0)
-                    elif m.group(1)!=None:
-                        addrequest(m.group(2), int(m.group(1)))
-                    elif m.group(3)!=None:
-                        inrequest = True
-                        reqpri = int(m.group(3))
-                        reqbinary = None
-                        reqoptions = []
-                        reqinfile = None
-    except IOError, (errno, strerror):
-        error(logpath, "could not be read: %s"%strerror)
-    else:
-        log.close()
-    return reqs
-
-re_error = re.compile('^! ')
-def haderrors(dirname, jobname):
-    logpath = os.path.join(dirname, jobname+".log")
-    if not os.path.isfile(logpath):
-        return None
-    try:
-        f = open(logpath)
-        for line in f:
-            if re_error.match(line):
-                return " (because of error in .log file)"
-            if re_logmatcher.match(line):
-                return " (because of request in .log file)"
-    except IOError, (errno, strerror):
-        error(logpath, "could not be read: %s"%strerror)
-    else:
-        f.close()
-    return False
-
 def rmtempfile(filenames, dirname, force=False, reason=""):
-    if opts.deletetempfiles or force:
+    if (opts.deletetempfiles or force) and filenames:
         if not isinstance(filenames, list):
             filenames = [filenames]
         if opts.verbosity > 1:
@@ -597,6 +731,17 @@ def run(arglist, dirname, reason="", inf=None, outf=None, stderr=None):
         except (OSError, IOError), e:
             error(dirname, "failed command: %s: %s"%(" ".join(arglist),str(e)))
 
+def runtex(tex, texname, realname, dirname, reason=""):
+    if not opts.showoutput:
+        outf = None
+    else:
+        def outf(d):
+            return re_rmligsfile.sub("\\1.tex", d)
+    if texname != realname:
+        run([tex, "-interaction", opts.interactionmode, "-jobname", texname[:-4], realname], dirname, outf=outf, reason=reason)
+    else:
+        run([tex, "-interaction", opts.interactionmode, texname], dirname, outf=outf, reason=reason)
+
 def preparermligs(texname, dirname, tmplist):
     try:
         texfile = open(os.path.join(dirname, texname))
@@ -637,44 +782,73 @@ def preparermligs(texname, dirname, tmplist):
                 rmligsfile.close()
             return rmligsname
 
-re_rmligsfile = re.compile("\\.([^/ ]+)\\.rmligs\\.tex")
-def runtex(tex, texname, realname, dirname, reason=""):
-    if not opts.showoutput:
-        outf = None
-    else:
-        def outf(d):
-            return re_rmligsfile.sub("\\1.tex", d)
-    if texname != realname:
-        run([tex, "-interaction", opts.interactionmode, "-jobname", texname[:-4], realname], dirname, outf=outf, reason=reason)
-    else:
-        run([tex, "-interaction", opts.interactionmode, texname], dirname, outf=outf, reason=reason)
-
 # main program:
-def main():
-    try:
-        if programargs:
-            processtexfiles(programargs)
+def main(arguments):
+    if not arguments:
+        arguments = (".",)
+
+    for dirname, texname in alltexfiles(arguments):
+        data = analyseTex(dirname, texname)
+        reason = needsupdate(dirname, texname, data)
+        if reason:
+            processTex(dirname, texname, data, reason)
+
+
+    if opts.summary in [ "files", "both" ] :
+        if procfiles:
+            print "Processed the following files:"
+            for f in procfiles:
+                print "    %s"%f
         else:
-            processtexfiles(["."])
+            print "Processed no files."
+    if opts.summary in [ "failures", "both" ] :
+        if errors:
+            print "The following problems occured:"
+            for f in errors:
+                print "    %s"%f
 
-        if opts.summary in [ "files", "both" ] :
-            if procfiles:
-                print "Processed the following files:"
-                for f in procfiles:
-                    print "    %s"%f
-            else:
-                print "Processed no files."
-        if opts.summary in [ "failures", "both" ] :
-            if errors:
-                print "The following problems occured:"
-                for f in errors:
-                    print "    %s"%f
+    if errors:
+        sys.exit(1)
 
-        if errors:
-            sys.exit(1)
+try:
+    if __name__ == "__main__":
+        # setup global variables
+        opts, programargs = TexallOptionParser().parse_args()
+    else:
+        opts, programargs = TexallOptionParser().parse_args([])
+
+    re_ignore = None
+    if opts.ignorepattern != "":
+        re_ignore = re.compile(opts.ignorepattern)
 
-    except KeyboardInterrupt:
-        sys.exit(2)
+    re_required = re.compile(opts.requiredpattern)
 
-if __name__ == "__main__":
-    main()
+    re_texfile = re.compile(".*\.tex$", re.I)
+
+    re_inputinclude = re.compile('\\\\(input|include|@input)\\{([^}]*)\\}')
+    re_usepackage = re.compile('\\\\usepackage(\\[.*?\\])?\\{([^}]*)\\}')
+    re_documentclass = re.compile('\\\\documentclass(\\[.*?\\])?\\{([^}]*)\\}')
+    re_graphics = re.compile('\\\\includegraphics(\\[.*?\\])?\\{([^}]*)\\}')
+    if iswindows:
+        re_systemfile = re.compile("^?:\\\\Program", re.I)
+    else:
+        re_systemfile = re.compile("^/usr/")
+    re_bibliography = re.compile('\\\\bibliography\\{([^}]*)\\}')
+    re_commawhitespace = re.compile('\\s*,\\s*')
+    re_needsps = re.compile('\\\\usepackage(\\[.*?\\])?\\{[^}]*pstricks[^}]*\\}')
+    re_rmligs = re.compile('\\\usepackage((\\[.*?\\])?\\{n?german\\}|\\[[^]]*?german[^]]*?\\]\\{babel\\})')
+
+    re_logmatcher = re.compile("^LaTeX Warning: There were undefined references\.$|^LaTeX Warning: Label\(s\) may have changed\. Rerun to get cross-references right\.$|^REQ:(\d+):(\w+):REQ|^REQ:(\d+):(\w+):$")
+
+    re_error = re.compile('^! ')
+    re_rmligsfile = re.compile("\\.([^/ ]+)\\.rmligs\\.tex")
+
+    if __name__ == "__main__":
+        main(programargs)
+
+except KeyboardInterrupt:
+    sys.exit(2)
+else:
+    for proc, master in kpseproc.values():
+        os.close(master)
+        os.kill(proc.pid, signal.SIGHUP)