#!/usr/bin/python
#
# adt-run is part of autopkgtest
# autopkgtest is a tool for testing Debian binary packages
#
# autopkgtest is Copyright (C) 2006-2007 Canonical Ltd.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
# See the file CREDITS for a full list of credits information (often
# installed as /usr/share/doc/autopkgtest/CREDITS).

import signal
import optparse
import tempfile
import sys
import subprocess
import traceback
import urllib
import string
import re
import os
import errno
import fnmatch
import shutil
import copy
import time

try:
    from debian import deb822, debian_support
    deb822, debian_support  # pyflakes
except ImportError:
    from debian_bundle import deb822, debian_support

from optparse import OptionParser

signal.signal(signal.SIGINT, signal.SIG_DFL)  # undo stupid Python SIGINT thing

try:
    our_base = os.environ['AUTOPKGTEST_BASE'] + '/lib'
except KeyError:
    our_base = '/usr/share/autopkgtest/python'
sys.path.insert(1, our_base)

from Autopkgtest import shellquote_cmdl

#---------- global variables

tmp = None		# pathstring on host
testbed = None		# Testbed
errorcode = 0		# exit status that we are going to use
timeouts = {'short': 100, 'copy': 300, 'install': 3000, 'test': 10000, 'build': 100000}
binaries = None		# Binaries (.debs we have registered)
build_essential = ["build-essential"]
paths = []
os.putenv("APT_LISTBUGS_FRONTEND", "none")
          # do not consider using apt-listbugs
os.putenv("APT_LISTCHANGES_FRONTEND", "none")
          # do not consider using apt-listchanges

#---------- output handling
#
#  There are at least the following kinds of output:
#
#   1. stderr output which consists of
#   1a. our errors
#   1b. stuff printed to stderr by the virtualisation server
#   1c. stuff printed to stderr by our short-lived subprocesseses
#           which we don't expect to fail
#
#   2. trace information, which consists of
#   2a. our trace information from calls to debug()
#   2b. progress information and stderr output from
#        general scripts we run on the host
#   2c. progress information and stderr output from things
#        we run on the testbed including builds
#   2d. stderr and stdout output from actual tests
#
#   xxxx
#   3. actual test results (printed to our stdout)
#
# Cloning of 1a and 2a, where necessary, is done by us writing
# the data twice.  Cloning of 1[bc] and 2[bc], where necessary,
# is done by forking off a copy of ourselves to do plumbing,
# which copy we wait for at the appropriate point.


class DummyOpts:

    def __init__(self):
        self.debuglevel = 0

opts = DummyOpts()
trace_stream = None
summary_stream = None


def pstderr(m):
    print >> sys.stderr, m
    if trace_stream is not None:
        print >> trace_stream, m


def debug(m, minlevel=0):
    if opts.debuglevel < minlevel:
        return
    if opts.quiet and trace_stream is None:
        return
    p = 'adt-run'
    if minlevel:
        p += str(minlevel)
    p += ': '
    for l in m.rstrip('\n').split('\n'):
        s = p + l
        if not opts.quiet:
            print >>sys.stderr, s
        if trace_stream is not None:
            print >>trace_stream, s


def debug_file(hp, minlevel=0):
    if opts.debuglevel < minlevel:
        return

    def do_copy(stream, what):
        rc = subprocess.call(['cat', hp], stdout=stream)
        if rc:
            bomb('cat failed copying data from %s'
                 ' to %s, exit code %d' % (hp, what, rc))
    if not opts.quiet:
        do_copy(sys.stderr, 'stderr')
    if trace_stream is not None:
        do_copy(trace_stream, 'trace log')


class Errplumb:

    def __init__(self, critical=False):
        to_stderr = critical or not opts.quiet
        count = to_stderr + (trace_stream is not None)
        if count == 0:
            self.stream = open('/dev/null', 'w')
            self._sp = None
        elif count == 1:
            if to_stderr:
                self.stream = os.dup(2)
            else:
                self.stream = trace_stream
            self._sp = None
        else:
            self._sp = subprocess.Popen(['tee', '-a', '/dev/stderr'],
                                        stdin=subprocess.PIPE,
                                        stdout=trace_stream,
                                        close_fds=True)
            self.stream = self._sp.stdin

    def wait(self):
        if self._sp is None:
            return
        if isinstance(self.stream, int):
            os.close(self.stream)
            self.stream = ()
        self._sp.stdin.close()
        rc = self._sp.wait()
        if rc:
            bomb('stderr plumbing tee(1) failed, exit code %d' % rc)
        self._sp = None


def subprocess_cooked(cmdl, critical=False, dbg=None, **kwargs):
    if dbg is not None:
        if isinstance(dbg, tuple):
            (what, script) = dbg
        else:
            (what, script) = (dbg, None)
        debug_subprocess(what, cmdl, script=script)
    ep = Errplumb(critical)
    running = subprocess.Popen(cmdl, stderr=ep.stream, **kwargs)
    output = running.communicate()[0]
    rc = running.wait()
    ep.wait()
    return (rc, output)


def psummary(m):
    if summary_stream is not None:
        print >>summary_stream, m


def preport(m):
    print m
    sys.stdout.flush()
    if trace_stream is not None:
        print >>trace_stream, m
    psummary(m)


def report(tname, result):
    preport('%-20s %s' % (tname, result.decode('UTF-8', 'replace')))

#---------- errors we define


class Quit:

    def __init__(self, ec, m):
        self.ec = ec
        self.m = m


def bomb(m):
    raise Quit(20, "unexpected error: %s" % m)


def badpkg(m):
    preport('blame: ' + ' '.join(testbed.blamed))
    preport('badpkg: ' + m)
    raise Quit(12, "erroneous package: %s" % m)


class Unsupported(Exception):

    def __init__(self, lno, m):
        if lno >= 0:
            self.m = '%s (control line %d)' % (m, lno)
        else:
            self.m = m

    def report(self, tname):
        global errorcode
        errorcode != 2
        report(tname, 'SKIP %s' % self.m)

#---------- convenience function


def mkdir_okexist(pathname, mode=02755):
    try:
        os.mkdir(pathname, mode)
    except (IOError, OSError), oe:
        if oe.errno != errno.EEXIST:
            raise


def rmtree(what, pathname):
    debug('/ %s rmtree %s' % (what, pathname), 2)
    try:
        shutil.rmtree(pathname)
    except (IOError, OSError), oe:
        if oe.errno != errno.EEXIST:
            raise


def debug_subprocess(what, cmdl=None, script=None):
    o = '$ ' + what + ':'
    if cmdl is not None:
        ol = []
        for x in cmdl:
            if x is script:
                x = '<SCRIPT>'
            ol.append(x.replace('\\', '\\\\').replace(' ', '\\ '))
        o += ' ' + ' '.join(ol)
    debug(o)
    if script is not None and opts.debuglevel >= 1:
        o = ''
        for l in script.rstrip('\n').split('\n'):
            o += '$     ' + l + '\n'
        debug(o, 1)


def flatten(l):
    return reduce((lambda a, b: a + b), l, [])

#---------- fancy automatic file-copying class


class AutoFile:
    # p.what
    # p.path[tb]	None or path	not None => path known
    # p.file[tb]	None or path	not None => file exists
    # p.ephem[tb]       boolean         True => destroyed by tb reset
    # p.spec	        string or None
    # p.spec_tbp	True or False, or not set if spec is None
    # p.dir		'' or '/'

    def __init__(self, what):
        self.what = what
        self.path = [None, None]
        self.file = [None, None]
        self.ephem = [False, False]
        self.spec = None
        self.spec_tbp = None
        self.dir = ''

    def __repr__(self):
        return "<AF@%s>" % self.__str__()

    def __str__(self):
        def ptbp(tbp):
            if self.path[tbp] is None:
                return '-' + self.dir
            elif self.file[tbp] is None:
                return self.path[tbp] + self.dir + '?'
            else:
                return self.path[tbp] + self.dir + '!'
        out = self.what
        if self.spec is not None:
            if not hasattr(self, 'spec_tb'):
                out += '~'
            elif self.spec_tbp:
                out += '#'
            else:
                out += '='
            out += self.spec
        out += ':'
        out += ptbp(False)
        out += '|'
        out += ptbp(True)
        return out

    def _wrong(self, how):
        xtra = ''
        if self.spec is not None:
            xtra = ' spec[%s]=%s' % (self.spec, getattr(self, 'spec_tb', None))
        raise Exception("internal error: %s (%s)%s" % (how, str(self), xtra))

    def _ensure_path(self, tbp):
        if self.path[tbp] is None:
            if '/' in self.what:
                self._debug('tmp-parent %s...' % 'HT'[tbp])
                TemporaryDir(os.path.dirname(self.what)).write(tbp)
            if not tbp:
                self.path[tbp] = tmp + '/' + self.what
            else:
                self.ephem[tbp] = True
                self.path[tbp] = testbed.scratch.path[True] + '/' + self.what

    def write(self, tbp=False):
        self._debug('write %s...' % 'HT'[tbp])
        self._ensure_path(tbp)

        if self.dir and not self.file[tbp]:
            if not tbp:
                self._debug('mkdir H')
                mkdir_okexist(self.path[tbp])
            else:
                cmdl = ['sh', '-ec',
                        'test -d "$1" || mkdir -p "$1"',
                        'x', self.path[tbp]]
                tf_what = urllib.quote(self.what).replace('/', '%2F')
                rc = testbed.execute('mkdir-' + tf_what, cmdl)
                if rc:
                    bomb('failed to create directory %s' %
                         self.path[tbp])

        self.file[tbp] = self.path[tbp]
        return self.path[tbp]

    def read(self, tbp=False):
        self._debug('read %s...' % 'HT'[tbp])
        self._ensure_path(tbp)

        if self.file[tbp] is None:
            if self.file[not tbp] is None:
                self._wrong("requesting read but nonexistent")
            cud = ['copyup', 'copydown'][tbp]
            src = self.file[not tbp] + self.dir
            dst = self.path[tbp] + self.dir
            testbed.command(cud, (src, dst))
            self.file[tbp] = self.path[tbp]

        return self.file[tbp] + self.dir

    def invalidate(self, tbp=False):
        if self.path[tbp] is not None:
            self._debug('invalidating %s' % 'HT'[tbp])
        self.file[tbp] = None
        if self.spec_tbp != tbp or self.spec is None:
            self.path[tbp] = None

    def invalidate_ephem(self, tbp=False):
        if self.ephem[tbp]:
            self.invalidate(tbp)

    def _debug(self, m):
        debug('/ %s#%x: %s' % (self.what, id(self), m), 3)

    def _constructed(self):
        self._debug('constructed: ' + str(self))
        self._check()
        paths.append(self)

    def _check(self):
        for tbp in [False, True]:
            for pf in [self.path, self.file]:
                if pf[tbp] is None:
                    continue
                if not pf[tbp]:
                    bomb('empty path specified for %s' % self.what)
                if self.dir and pf[tbp].endswith('/'):
                    pf[tbp] = pf[tbp].rstrip('/')
                    if not pf[tbp]:
                        pf[tbp] = '/'
                if not self.dir and pf[tbp].endswith('/'):
                    bomb("directory `%s' specified for "
                         "non-directory %s" % (pf[tbp], self.what))

    def _relative_init(self, what, parent, leaf, onlyon_tbp, setfiles, sibling):
        AutoFile.__init__(self, what)
        sh_on = ''
        sh_sibl = ''
        if onlyon_tbp is not None:
            sh_on = ' (on %s)' % ('HT'[onlyon_tbp])
        if sibling:
            sh_sibl = ' (sibling)'
        parent._debug('using as base: %s: %s%s%s' %
                      (str(parent), leaf, sh_on, sh_sibl))
        if not sibling and not parent.dir:
            parent._wrong('asked for non-sibling relative path of non-dir')
        if sibling:
            trim = os.path.dirname
        else:
            trim = lambda x: x
        for tbp in [False, True]:
            self.ephem[tbp] = parent.ephem[tbp]
            if parent.path[tbp] is None:
                continue
            trimmed = trim(parent.path[tbp])
            if trimmed:
                trimmed += '/'
            self.path[tbp] = trimmed + leaf
            if setfiles and (onlyon_tbp is None or onlyon_tbp == tbp):
                self.file[tbp] = self.path[tbp]


class InputFile(AutoFile):

    def _init(self, what, spec, spec_tbp=False):
        AutoFile.__init__(self, what)
        self.spec = spec
        self.spec_tbp = spec_tbp
        self.path[spec_tbp] = spec
        self.file[self.spec_tbp] = self.path[self.spec_tbp]

    def __init__(self, what, spec, spec_tbp=False):
        self._init(what, spec, spec_tbp)
        self._constructed()


class InputDir(InputFile):

    def __init__(self, what, spec, spec_tbp=False):
        InputFile._init(self, what, spec, spec_tbp)
        self.dir = '/'
        self._constructed()


class OutputFile(AutoFile):

    def _init(self, what, spec, spec_tbp=False):
        AutoFile.__init__(self, what)
        self.spec = spec
        self.spec_tbp = spec_tbp
        self.path[spec_tbp] = spec

    def __init__(self, what, spec, spec_tbp=False):
        self._init(what, spec, spec_tbp)
        self._constructed()


class OutputDir(OutputFile):

    def __init__(self, what, spec, spec_tbp=False):
        OutputFile._init(self, what, spec, spec_tbp)
        self.dir = '/'
        self._constructed()


class RelativeInputDir(AutoFile):

    def __init__(self, what, parent, leaf, onlyon_tbp=None, sibling=False):
        self._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
        self.dir = '/'
        self._constructed()


class RelativeInputFile(AutoFile):

    def __init__(self, what, parent, leaf, onlyon_tbp=None, sibling=False):
        self._relative_init(what, parent, leaf, onlyon_tbp, True, sibling)
        self._constructed()


class RelativeOutputFile(AutoFile):

    def __init__(self, what, parent, leaf, sibling=False):
        self._relative_init(what, parent, leaf, None, False, sibling)
        self._constructed()


class TemporaryFile(AutoFile):

    def __init__(self, what):
        AutoFile.__init__(self, what)
        self._constructed()


class TemporaryDir(AutoFile):

    def __init__(self, what):
        AutoFile.__init__(self, what)
        self.dir = '/'
        self._constructed()

#---------- parsing and representation of the arguments


class Action:

    def __init__(self, kind, af, arghandling, what):
        # extra attributes get added during processing
        self.kind = kind
        self.af = af
        self.ah = arghandling
        self.what = what
        self.missing_tests_control = False

    def __repr__(self):
        return "<Action %s %s %s>" % (self.kind, self.what, repr(self.af))


def parse_args():
    global opts, timeouts
    global n_non_actions  # argh, stupid python scoping rules
    usage = "%prog <options> --- <virt-server>..."
    parser = OptionParser(usage=usage)
    pa = parser.add_option

    arghandling = {
        'dsc_tests': True,
        'dsc_filter': '*',
        'deb_forbuilds': 'auto',
        'deb_fortests': 'auto',
        'tb': False,
        'override_control': None,
        'set_lang': 'C'
    }
    initial_arghandling = arghandling.copy()
    n_non_actions = 0

    #----------
    # actions (ie, test sets to run, sources to build, binaries to use):

    def cb_action(op, optstr, value, parser, long, kindpath, is_act):
        global n_non_actions
        parser.largs.append((value, kindpath))
        n_non_actions += not(is_act)

    def pa_action(long, metavar, kindpath, help, is_act=True):
        pa('', '--' + long, action='callback', callback=cb_action,
           nargs=1, type='string',
           callback_args=(long, kindpath, is_act), help=help)

    pa_action('built-tree', 'TREE', '@/',
              help='run tests from build tree TREE')

    pa_action('unbuilt-tree', 'TREE', '@//',
              help='run tests from build tree TREE')

    pa_action('source', 'DSC', '@.dsc',
              help='build DSC and use its tests and/or'
              ' generated binary packages')

    pa_action('binary', 'DEB', '@.deb',
              help='use binary package DEB according'
              ' to most recent --binaries-* settings')

    def cb_actnoarg(op, optstr, value, parser, largsentry):
        parser.largs.append(largsentry)
    pa('', '--instantiate', action='callback', callback=cb_actnoarg,
       callback_args=((None, ('instantiate',)),),
       help='instantiate testbed now (during testing phase)'
       ' and install packages'
       ' selected for automatic installation, even'
       ' if this might apparently not be required otherwise')

    pa_action('override-control', 'CONTROL', ('control',), is_act=0,
              help='run tests from control file CONTROL instead,'
              ' (applies to next test suite only)')

    #----------
    # argument handling settings (what ways to use action
    #  arguments, and pathname processing):

    def cb_setah(option, opt_str, value, parser, toset, setval):
        if type(setval) == list:
            if not value in setval:
                parser.error('value for %s option (%s) is not '
                             'one of the permitted values (%s)' %
                             (value, opt_str, setval.join(' ')))
        elif setval is not None:
            value = setval
        for v in toset:
            arghandling[v] = value
        parser.largs.append(arghandling.copy())

    def pa_setah(long, affected, effect, metavar=None, **kwargs):
        type = metavar
        if type is not None:
            type = 'string'
        pa('', long, action='callback', callback=cb_setah,
           callback_args=(affected, effect), **kwargs)

    #---- paths: host or testbed:
    #
    pa_setah('--paths-testbed', ['tb'], True,
             help='subsequent path specifications refer to the testbed')
    pa_setah('--paths-host', ['tb'], False,
             help='subsequent path specifications refer to the host')

    #---- source processing settings:

    pa_setah('--sources-tests', ['dsc_tests'], True,
             help='run tests from builds of subsequent sources')
    pa_setah('--sources-no-tests', ['dsc_tests'], False,
             help='do not run tests from builds of subsequent sources')

    pa_setah('--built-binaries-filter', ['dsc_filter'], None,
             type='string', metavar='PATTERN-LIST',
             help='from subsequent sources, use binaries matching'
             ' PATTERN-LIST (comma-separated glob patterns)'
             ' according to most recent --binaries-* settings')
    pa_setah('--no-built-binaries', ['dsc_filter'], '_',
             help='from subsequent sources, do not use any binaries')
    #---- binary package processing settings:

    def pa_setahbins(long, toset, how):
        pa_setah(long, toset, ['ignore', 'auto', 'install'],
                 type='string', metavar='IGNORE|AUTO|INSTALL', default='auto',
                 help=how + ' ignore binaries, install them as needed'
                 ' for dependencies, or unconditionally install'
                 ' them, respectively')
    pa_setahbins('--binaries', ['deb_forbuilds', 'deb_fortests'], '')
    pa_setahbins('--binaries-forbuilds', ['deb_forbuilds'], 'for builds, ')
    pa_setahbins('--binaries-fortests', ['deb_fortests'], 'for tests, ')

    #----------
    # general options:

    def cb_vserv(op, optstr, value, parser):
        parser.values.vserver = list(parser.rargs)
        del parser.rargs[:]

    def cb_path(op, optstr, value, parser, constructor, long, dir):
        name = long.replace('-', '_')
        af = constructor(long, value, arghandling['tb'])
        setattr(parser.values, name, af)

    def pa_path(long, constructor, help, dir=False):
        pa('', '--' + long, action='callback', callback=cb_path,
           callback_args=(constructor, long, dir),
           nargs=1, type='string',
           help=help, metavar='PATH')

    pa_path('output-dir', OutputDir, dir=True,
            help='write stderr/out files in PATH')

    pa('--leave-lang', dest='set_lang', action='store_false',
       help="leave LANG on testbed set to testbed's default")
    pa('--set-lang', dest='set_lang', action='store', metavar='LANGVAL',
       help='set LANG on testbed to LANGVAL', default='C')

    pa('', '--tmp-dir', type='string', dest='tmp',
       help='write temporary files to TMP, emptying it'
       ' beforehand and leaving it behind at the end')
    pa('', '--log-file', type='string', dest='logfile',
       help='write the log LOGFILE, emptying it beforehand,'
       ' instead of using OUTPUT-DIR/log or TMPDIR/log')
    pa('', '--summary-file', type='string', dest='summary',
       help='write a summary report to SUMMARY,'
       ' emptying it beforehand')

    for k in timeouts.keys():
        pa('', '--timeout-' + k, type='int', dest='timeout_' + k,
           metavar='T', help='set %s timeout to T' % k)
    pa('', '--timeout-factor', type='float', dest='timeout_factor',
       metavar='FACTOR', default=1.0,
       help='multiply all default timeouts by FACTOR')

    pa('', '--user', type='string', dest='user',
       help='run tests as USER (needs root on testbed)')
    pa('', '--gain-root', type='string', dest='gainroot',
       help='prefix debian/rules binary with GAINROOT')
    pa('-q', '--quiet', action='store_true', dest='quiet', default=False)
    pa('-d', '--debug', action='count', dest='debuglevel', default=0)
    pa('', '--gnupg-home', type='string', dest='gnupghome',
       default='~/.autopkgtest/gpg',
       help='use GNUPGHOME rather than ~/.autopkgtest (for'
       " signing private apt archive);"
       " `fresh' means generate new key each time.")

    #----------
    # actual meat:

    class SpecialOption(optparse.Option):
        pass
    vs_op = SpecialOption('', '--VSERVER-DUMMY')
    vs_op.action = 'callback'
    vs_op.type = None
    vs_op.default = None
    vs_op.nargs = 0
    vs_op.callback = cb_vserv
    vs_op.callback_args = ()
    vs_op.callback_kwargs = {}
    vs_op.help = 'introduces virtualisation server and args'
    vs_op._short_opts = []
    vs_op._long_opts = ['---']

    pa(vs_op)

    (opts, args) = parser.parse_args()
    if not hasattr(opts, 'vserver'):
        parser.error('you must specifiy --- <virt-server>...')
    if n_non_actions >= len(parser.largs):
        parser.error('nothing to do specified')

    for k in timeouts.keys():
        t = getattr(opts, 'timeout_' + k)
        if t is None:
            t = timeouts[k] * opts.timeout_factor
        timeouts[k] = int(t)

    # this timeout is for adt-virt-*, so pass it down via environment
    os.environ['ADT_VIRT_COPY_TIMEOUT'] = str(timeouts['copy'])

    arghandling = initial_arghandling
    opts.actions = []
    ix = 0
    for act in args:
        if type(act) == dict:
            arghandling = act
            continue
        elif type(act) == tuple:
            pass
        elif type(act) == str:
            act = (act, act)
        else:
            raise Exception("unknown action in list `%s' having"
                            " type `%s'" % (act, type(act)))
        (pathstr, kindpath) = act

        constructor = InputFile
        if type(kindpath) is tuple:
            kind = kindpath[0]
        elif kindpath.endswith('.deb'):
            kind = 'deb'
        elif kindpath.endswith('.dsc'):
            kind = 'dsc'
        elif kindpath.endswith('//'):
            kind = 'ubtree'
            constructor = InputDir
        elif kindpath.endswith('/'):
            kind = 'tree'
            constructor = InputDir
        else:
            parser.error("do not know how to handle filename `%s';"
                         " specify --source --binary or --build-tree" %
                         kindpath)

        what = '%s%s' % (kind, ix)
        ix += 1

        if kind == 'dsc':
            fwhatx = '/' + os.path.basename(pathstr)
        else:
            fwhatx = '-' + kind

        if pathstr is None:
            af = None
        else:
            af = constructor(what + fwhatx, pathstr, arghandling['tb'])
        opts.actions.append(Action(kind, af, arghandling, what))


def setup_trace():
    global trace_stream, tmp, summary_stream

    if opts.tmp is not None:
        rmtree('tmp(specified)', opts.tmp)
        mkdir_okexist(opts.tmp, 0755)
        tmp = opts.tmp
    else:
        assert(tmp is None)
        tmp = tempfile.mkdtemp()
        os.chmod(tmp, 0755)

    if opts.logfile is None:
        if opts.output_dir is not None and opts.output_dir.spec_tbp:
            opts.logfile = opts.output_dir.spec + '/log'
        elif opts.tmp is not None:
            opts.logfile = opts.tmp + '/log'
    if opts.logfile is not None:
        trace_stream = open(opts.logfile, 'w', 0)
    if opts.summary is not None:
        summary_stream = open(opts.summary, 'w', 0)

    debug('options: ' + str(opts) + '; timeouts: ' + str(timeouts), 1)


def finalise_options():
    global opts, tb, build_essential

    if opts.user is None and 'root-on-testbed' not in testbed.caps:
        opts.user = ''

    if opts.user is None:
        su = 'suggested-normal-user='
        ul = [
            e[len(su):]
            for e in testbed.caps
            if e.startswith(su)
        ]
        if ul:
            opts.user = ul[0]
        else:
            opts.user = ''

    if opts.user:
        if 'root-on-testbed' not in testbed.caps:
            pstderr("warning: virtualisation"
                    " system does not offer root on testbed,"
                    " but --user option specified: failure likely")
        opts.user_wrap = lambda x: "su %s -c '%s'" % (opts.user, x)
    else:
        opts.user_wrap = lambda x: x

    if opts.gainroot is None:
        opts.gainroot = ''
        if (opts.user or
                'root-on-testbed' not in testbed.caps):
            opts.gainroot = 'fakeroot'
            build_essential += ['fakeroot']

    if opts.gnupghome.startswith('~/'):
        opts.gnupghome = os.path.expanduser(opts.gnupghome)
    elif opts.gnupghome == 'fresh':
        opts.gnupghome = None

#---------- testbed management - the Testbed class


class Testbed:

    def __init__(self):
        self.sp = None
        self.lastsend = None
        self.scratch = None
        self.modified = False
        self.blamed = []
        self._debug('init', 1)
        self._need_reset_apt = False

    def _debug(self, m, minlevel=0):
        debug('** ' + m, minlevel)

    def start(self):
        self._debug('start', 1)
        debug_subprocess('vserver', opts.vserver)
        self._errplumb = Errplumb(True)
        self.sp = subprocess.Popen(opts.vserver,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=self._errplumb.stream)
        self.expect('ok')
        self.caps = self.commandr('capabilities')

    def stop(self):
        self._debug('stop', 1)
        self.close()
        if self.sp is None:
            return
        ec = self.sp.returncode
        if ec is None:
            self.sp.stdout.close()
            self.send('quit')
            self.sp.stdin.close()
            ec = self.sp.wait()
        if ec:
            self.bomb('testbed gave exit status %d after quit' % ec)
        self._errplumb.wait()
        self.sp = None

    def open(self):
        self._debug('open, scratch=%s' % self.scratch, 1)
        if self.scratch is not None:
            return
        pl = self.commandr('open')
        self._opened(pl)

    def _opened(self, pl):
        self.scratch = InputDir('tb-scratch', pl[0], True)
        self.deps_processed = []
        for af in paths:
            af.invalidate_ephem(True)
        self._auxverbscript_make()

    def _auxverbscript_make(self):
        pec = self.commandr('print-auxverb-command')
        if len(pec) < 1:
            self.bomb('too few results from print-execute-command')
        cmdl = map(urllib.unquote, pec[0].split(','))

        self._debug('cmdl = %s' % str(cmdl), 1)

        self.ec_auxverbscript = TemporaryFile('satdep-auxverb')
        print >>open(self.ec_auxverbscript.write(), 'w'), (
            '''#!/bin/sh
set -e
if [ $# = 2 ] && [ "x$1" = xdpkg-architecture ] && [ "x$2" = x-qDEB_HOST_ARCH ]; then
    # This is a pretty nasty hack.  Hopefully it can go away
    #  eventually.  See #635763.
    set -- dpkg --print-architecture
fi
export DEBIAN_FRONTEND=noninteractive
exec ''' + shellquote_cmdl(cmdl) + ' "$@"' + "\n"
        )
        os.chmod(self.ec_auxverbscript.write(), 0755)

    def mungeing_apt(self):
        if not 'revert' in self.caps:
            self._need_reset_apt = True

    def reset_apt(self):
        if not self._need_reset_apt:
            return
        what = 'aptget-update-reset'
        cmdl = ['sh', '-c', 'apt-get -qy update 2>&1']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            pstderr("\n" "warning: failed to restore"
                    " testbed apt cache, exit code %d" % rc)
        what = 'aptconf-reset'
        cmdl = ['rm', '-f', '/etc/apt/apt.conf.d/90autopkgtest',
                '/etc/apt/sources.list.d/autopkgtest.list',
                '/etc/apt/preferences.d/90autopkgtest']
        rc = self.execute(what, cmdl, kind='install')
        if rc:
            pstderr("\n" "warning: failed to reset changes"
                    " made to testbed apt configuration, exit code %d" % rc)
        self._need_reset_apt = False

    def close(self):
        self._debug('close, scratch=%s' % self.scratch, 1)
        if self.scratch is None:
            return
        self.scratch = None
        if self.sp is None:
            return
        self.command('close')

    def prepare1(self, deps_new):
        self._debug('prepare1, modified=%s, deps_processed=%s, deps_new=%s' %
                    (self.modified, self.deps_processed, deps_new), 1)
        if 'revert' in self.caps and (
                self.modified or [d for d in self.deps_processed if d not in deps_new]):
            for af in paths:
                af.read(False)
            self._debug('reset **', 1)
            pl = self.commandr('revert')
            self._opened(pl)
        self.modified = False

    def prepare2(self, deps_new):
        self._debug('prepare2, deps_new=%s' % deps_new, 1)
        binaries.publish()
        self._install_deps(deps_new)

    def prepare(self, deps_new):
        self.prepare1(deps_new)
        self.prepare2(deps_new)

    def _install_deps(self, deps_new):
        self._debug(' installing dependencies ' + str(deps_new), 1)
        self.deps_processed = deps_new
        if not deps_new:
            return
        self.satisfy_dependencies_string(', '.join(deps_new), 'install-deps')

    def needs_reset(self):
        self._debug('needs_reset, previously=%s' % self.modified, 1)
        self.modified = True

    def blame(self, m):
        self._debug('blame += %s' % m, 1)
        self.blamed.append(m)

    def bomb(self, m):
        self._debug('bomb %s' % m, 1)
        self.reset_apt()
        self.stop()
        raise Quit(16, 'testbed failed: %s' % m)

    def send(self, string):
        self.sp.stdin
        try:
            debug('>> ' + string, 2)
            print >>self.sp.stdin, string
            self.sp.stdin.flush()
            self.lastsend = string
        except:
            (type, value, dummy) = sys.exc_info()
            self.bomb('cannot send to testbed: %s' % traceback.
                      format_exception_only(type, value))

    def expect(self, keyword, nresults=None):
        l = self.sp.stdout.readline()
        if not l:
            self.bomb('unexpected eof from the testbed')
        if not l.endswith('\n'):
            self.bomb('unterminated line from the testbed')
        l = l.rstrip('\n')
        debug('<< ' + l, 2)
        ll = l.split()
        if not ll:
            self.bomb('unexpected whitespace-only line from the testbed')
        if ll[0] != keyword:
            if self.lastsend is None:
                self.bomb("got banner `%s', expected `%s...'" %
                          (l, keyword))
            else:
                self.bomb("sent `%s', got `%s', expected `%s...'" %
                          (self.lastsend, l, keyword))
        ll = ll[1:]
        if nresults is not None and len(ll) != nresults:
            self.bomb("sent `%s', got `%s' (%d result parameters),"
                      " expected %d result parameters" %
                      (self.lastsend, l, len(ll), nresults))
        return ll

    def commandr(self, cmd, args=(), nresults=None, unquote=True):
        # pass args=[None,...] or =(None,...) to avoid more url quoting
        if type(cmd) is str:
            cmd = [cmd]
        if len(args) and args[0] is None:
            args = args[1:]
        else:
            args = map(urllib.quote, args)
        al = cmd + args
        self.send(string.join(al))
        ll = self.expect('ok', nresults)
        if unquote:
            ll = map(urllib.unquote, ll)
        return ll

    def command(self, cmd, args=()):
        self.commandr(cmd, args, 0)

    def commandr1(self, cmd, args=()):
        rl = self.commandr(cmd, args, 1)
        return rl[0]

    def execute(self, what, cmdl,
                si='/dev/null', so='/dev/null', se=None, cwd=None,
                script=False, xenv=[], kind='short'):
        # Options for script:
        #   False - do not call debug_subprocess, no synch. reporting required
        #   None or string - call debug_subprocess with that value,
        #			plumb stderr through synchronously if possible
        # Options for se:
        #   None - usual Errplumb (output is of kind 2c)
        #   string - path on testbed (output is of kind 2d)

        timeout = timeouts[kind]

        if script is not False:
            debug_subprocess(what, cmdl, script=script)
        if cwd is None:
            cwd = self.scratch.write(True)

        xdump = None
        if se is None:
            ep = Errplumb()
            se_catch = TemporaryFile(what + '-xerr')
            se_use = se_catch.write(True)
            if not opts.quiet:
                xdump = 'debug=2-2'
            elif trace_stream is not None:
                xdump = 'debug=2-%d' % trace_stream.fileno()
        else:
            ep = None
            se_catch = None
            se_use = se

        cmdl = [None,
                ','.join(map(urllib.quote, cmdl)),
                si, so, se_use, cwd]

        if timeout is not None and timeout > 0:
            cmdl.append('timeout=%d' % timeout)

        if xdump is not None and 'execute-debug' in self.caps:
            cmdl += [xdump]
        for e in xenv:
            cmdl.append('env=%s' % e)
        if kind == 'install':
            cmdl.append('env=DEBIAN_FRONTEND=noninteractive')
        if opts.set_lang is not False:
            cmdl.append('env=LANG=%s' % opts.set_lang)

        rc = self.commandr1('execute', cmdl)
        try:
            rc = int(rc)
        except ValueError:
            bomb("execute for %s gave invalid response `%s'"
                 % (what, rc))

        if se_catch is not None:
            debug_file(se_catch.read())
        if ep is not None:
            ep.wait()

        return rc

    def satisfy_dependencies_string(self, deps, what):
        # Must have called Binaries.configure_apt
        debug('dependencies: %s: satisfying %s' % (what, deps), 1)
        dsc = TemporaryFile('deps.dsc')
        print >>open(dsc.write(), 'w'), 'Build-Depends: ', deps, '\n\n'
        # pbuilder-satisfydepends has a bug where it ignores the
        # Build-Depends if it's the last line in the dsc (#635696)
        self.satisfy_dependencies_dsc(dsc, what)

    def satisfy_dependencies_dsc(self, dsc, what):
        # Must have called Binaries.configure_apt
        cmdl = ['/usr/lib/pbuilder/pbuilder-satisfydepends-classic',
                '--binary-all',  # --check-key
                '--internal-chrootexec', self.ec_auxverbscript.read(),
                '-c', dsc.read()
                ]
        # The --internal-chrootexec option is really handy but
        # perhaps we are not supposed to use it ?  See also #635697.
        debug('dependencies: %s: running %s' % (what, str(cmdl)))
        rc = subprocess.call(cmdl, stdout=None, stderr=None)
        if rc:
            badpkg('dependency install failed, exit code %d' % rc)

#---------- representation of test control files: Field*, Test, etc.


class FieldBase:

    def __init__(self, fname, stz, base, tnames, vl):
        assert(vl)
        self.stz = stz
        self.fname = fname
        self.base = base
        self.tnames = tnames
        self.vl = vl

    def words(self):
        def distribute(vle):
            (lno, v) = vle
            r = v.split()
            r = map((lambda w: (lno, w)), r)
            return r
        return flatten(map(distribute, self.vl))

    def atmostone(self):
        if len(self.vl) == 1:
            (self.lno, self.v) = self.vl[0]
        else:
            raise Unsupported(self.vl[1][0],
                              'only one %s field allowed' % self.fname)
        return self.v


class FieldIgnore(FieldBase):

    def parse(self):
        pass


class Restriction:

    def __init__(self, rname, base):
        pass


class Restriction_rw_build_tree(Restriction):
    pass


class Restriction_build_needed(Restriction):
    pass


class Restriction_allow_stderr(Restriction):
    pass


class Restriction_breaks_testbed(Restriction):

    def __init__(self, rname, base):
        if 'revert-full-system' not in testbed.caps:
            raise Unsupported(-1, 'Test breaks testbed but testbed does not '
                              'advertise revert-full-system')


class Restriction_needs_root(Restriction):

    def __init__(self, rname, base):
        if 'root-on-testbed' not in testbed.caps:
            raise Unsupported(-1,
                              'Test needs root on testbed which is not available')


class Field_Restrictions(FieldBase):

    def parse(self):
        for wle in self.words():
            (lno, rname) = wle
            nrname = rname.replace('-', '_')
            try:
                rclass = globals()['Restriction_' + nrname]
            except KeyError:
                raise Unsupported(lno,
                                  'unknown restriction %s' % rname)
            r = rclass(nrname, self.base)
            self.base['restriction_names'].append(rname)
            self.base['restrictions'].append(r)


class Field_Features(FieldIgnore):

    def parse(self):
        for wle in self.words():
            (lno, fname) = wle
            self.base['feature_names'].append(fname)
            nfname = fname.replace('-', '_')
            try:
                fclass = globals()['Feature_' + nfname]
            except KeyError:
                continue
            ft = fclass(nfname, self.base)
            self.base['features'].append(ft)


class Field_Tests(FieldIgnore):
    pass


class Field_Depends(FieldBase):

    def parse(self):
        debug("Field_Depends: %s %s %s %s" %
              (self.stz, self.base, self.tnames, self.vl), 2)
        dl = map(lambda x: x.strip(),
                 flatten(map(lambda (lno, v): v.split(','), self.vl)))
        # allow empty Depends: field
        if dl == ['']:
            self.base['depends'] = []
            return

        dep_re = re.compile(
            r"(?P<package>[a-z0-9+-.]+)\s*(\((?P<relation><<|<=|>=|=|>>)\s*(?P<version>[^\)]*)\))?(\s*\[[[a-z0-9+-. ]+\])?$")
        for di in dl:
            for d in di.split('|'):
                d = d.strip()
                if d == '@':
                    continue  # Expanded to binary packages
                m = dep_re.match(d)
                if not m:
                    badpkg("Test Depends field contains an invalid "
                           "dependency `%s'" % d)
                if m.group("version"):
                    try:
                        debian_support.NativeVersion(m.group("version"))
                    except ValueError:
                        badpkg("Test Depends field contains dependency"
                               " `%s' with an invalid version" % d)
        self.base['depends'] = dl


class Field_Tests_directory(FieldBase):

    def parse(self):
        td = self.atmostone()
        if td.startswith('/'):
            raise Unsupported(self.lno,
                              'Tests-Directory may not be absolute')
        self.base['testsdir'] = td


def run_tests(stanzas, tree):
    global errorcode, testbed
    if stanzas == ():
        report('*', 'SKIP no tests in this package')
        errorcode |= 8
    for stanza in stanzas:
        tests = stanza[' tests']
        if not tests:
            report('*', 'SKIP package has metadata but no tests')
            errorcode |= 8
        for t in tests:
            t.prepare(tree)
            t.run(tree)
            if 'breaks-testbed' in t.restriction_names:
                testbed.needs_reset()
        testbed.needs_reset()


class Test:

    def __init__(self, tname, base, act):
        if '/' in tname:
            raise Unsupported(base[' lno'],
                              'test name may not contain / character')
        for k in base:
            setattr(self, k, base[k])
        self.tname = tname
        self.act = act
        self.what = act.what + 't-' + tname
        if len(base['testsdir']):
            self.path = base['testsdir'] + '/' + tname
        else:
            self.path = tname
        debug('constructed; path=%s' % self.path, 1)
        debug(' .depends=%s' % self.depends, 1)

    def _debug(self, m):
        debug('& %s: %s' % (self.what, m))

    def report(self, m):
        report(self.what, m)

    def reportfail(self, m):
        global errorcode
        errorcode |= 4
        report(self.what, 'FAIL ' + m)

    def prepare(self, tree):
        self._debug('preparing')
        dn = []
        for d in self.depends:
            self._debug(' processing dependency ' + d)
            if not '@' in d:
                self._debug('  literal dependency ' + d)
                dn.append(d)
            else:
                for pkg in packages_from_source(self.act, tree):
                    dp = d.replace('@', pkg)
                    self._debug('  synthesised dependency ' + dp)
                    dn.append(dp)
        testbed.prepare(dn)

    def run(self, tree):
        self._debug('[----------------------------------------')

        def stdouterr(oe):
            idstr = self.what + '-' + oe
            if opts.output_dir is not None and opts.output_dir.spec_tbp:
                use_dir = opts.output_dir
            else:
                use_dir = testbed.scratch
            return RelativeOutputFile(idstr, use_dir, idstr)

        if hasattr(self.act, 'work'):
            self.act.work.write(True)
        tree.read(True)

        af = RelativeInputFile(self.what, tree, self.path)
        so = stdouterr('stdout')
        se = stdouterr('stderr')

        tf = af.read(True)
        xenv = []

        rc = testbed.execute('testchmod-' + self.what, ['chmod', '+x', '--', tf])
        if rc:
            bomb('failed to chmod +x %s' % tf)

        testtmp = '%s%s-testtmp' % (testbed.scratch.read(True), self.what)

        script = 'buildtree="$1"; shift\n'
        script += 'rm -rf -- "$@"; mkdir -- "$@"\n'

        if 'needs-root' not in self.restriction_names and opts.user is not None:
            tfl = ['su', opts.user, '-c', tf]
            if opts.user:
                script += 'chown %s "$@"\n' % opts.user
                if 'rw-build-tree' in self.restriction_names:
                    script += ('chown -R %s "$buildtree"\n'
                               % opts.user)
        else:
            tfl = [tf]

        test_tmpdir = testtmp + \
            '/tmpdir'
        xenv.append('TMPDIR=%s' % test_tmpdir)
        test_adttmp = testtmp + \
            '/adttmp'
        xenv.append('ADTTMP=%s' % test_adttmp)
        script += 'chmod 1777 "%s"\n' % test_tmpdir
        rc = testbed.execute('mktmpdir-' + self.what,
                             ['sh', '-ec', script, 'x', tree.read(True),
                              testtmp, test_tmpdir, test_adttmp,
                              ])
        if rc:
            bomb("could not create test tmp dirs in `%s', exit code %d"
                 % (testtmp, rc))

        # if testtmp is a local dir, we are using the null runner and can show
        # stdout and stderr in realtime by teeing them (we can't use the
        # coreutils tee with pipes though, as it aborts on the first read
        # error)
        show_realtime = os.path.isdir(testtmp)
        if show_realtime:
            so_write = so.write(True)
            se_write = se.write(True)
            fifo_out = os.path.join(testtmp, 'test_stdout')
            fifo_err = os.path.join(testtmp, 'test_stderr')
            os.mkfifo(fifo_out)
            os.mkfifo(fifo_err)
            debug('teeing to stdout: %s, stderr: %s' % (fifo_out, fifo_err), 1)
            tee_out = os.fork()
            if tee_out == 0:
                fd_in = os.open(fifo_out, os.O_RDONLY)
                fd_out = os.open(so_write, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(1, block)
                os._exit(0)

            tee_err = os.fork()
            if tee_err == 0:
                fd_in = os.open(fifo_err, os.O_RDONLY)
                fd_out = os.open(se_write, os.O_CREAT | os.O_WRONLY)

                while True:
                    block = os.read(fd_in, 1024)
                    if not block:
                        break
                    os.write(fd_out, block)
                    os.write(2, block)
                os._exit(0)

            try:
                rc = testbed.execute('test-' + self.what, tfl,
                                     so=fifo_out, se=fifo_err, cwd=tree.read(True),
                                     xenv=xenv, kind='test')
                debug('testbed executing test finished with exit status %i' % rc, 1)
            finally:
                # give tee childs another second to mop up last output
                time.sleep(1)
                os.kill(tee_out, signal.SIGTERM)
                os.kill(tee_err, signal.SIGTERM)
                os.waitpid(tee_out, 0)
                os.waitpid(tee_err, 0)

            os.unlink(fifo_out)
            os.unlink(fifo_err)

        # if testtmp is not accessible, we cannot show realtime output and need
        # to show stdout afterwards
        else:
            rc = testbed.execute('test-' + self.what, tfl,
                                 so=so.write(True), se=se.write(True), cwd=tree.read(True),
                                 xenv=xenv, kind='test')
            debug('testbed executing test finished with exit status %i' % rc, 1)

        self._debug('----------------------------------------]')

        so_read = so.read()
        se_read = se.read()

        self._debug(' - - - - - - - - - - results - - - - - - - - - -')

        stderr_stat = os.stat(se_read)

        if rc != 0:
            self.reportfail('non-zero exit status %d' % rc)
        elif (stderr_stat.st_size != 0 and
              'allow-stderr' not in self.restriction_names):
            se_read_fobj = open(se_read)
            stderr_top = se_read_fobj.readline().rstrip('\n \t\r')
            se_read_fobj.close()
            self.reportfail('status: %d, stderr: %s' % (rc, stderr_top))
            self._debug(' - - - - - - - - - - stderr - - - - - - - - - -')
            debug_file(se_read)
        else:
            self.report('PASS')

        if not show_realtime:
            if os.stat(so_read).st_size != 0:
                self._debug(' - - - - - - - - - - stdout - - - - - - - - - -')
                debug_file(so_read)


def read_stanzas(af):
    stanzas = []

    try:
        control = open(af.read(), 'r')
    except (IOError, OSError), oe:
        if oe[0] != errno.ENOENT:
            raise
        return []

    lno = 0
    stz = {}  # stz[field_name][index] = (lno, value)
                # special field names:
                # stz[' lno'] = number
                # stz[' tests'] = list of Test objects
                # empty dictionary means we're between stanzas
    for paragraph in deb822.Deb822.iter_paragraphs(control):
        lno += 1
        stz = {' lno': lno, ' tests': []}
        for field, value in paragraph.iteritems():
            v = "".join(value.split('\n')).replace('  ', ' ')
            field = string.capwords(field)
            stz[field] = [(lno, v)]
            lno += 1 + value.count('\n')  # Count multilines fields
        stanzas.append(stz.copy())

    return stanzas


def read_control(act, tree, control_override):

    if control_override is not None:
        control_af = control_override
        testbed.blame('arg:' + control_override.spec)
    else:
        if act.missing_tests_control:
            return ()
        control_af = RelativeInputFile(act.what + '-testcontrol',
                                       tree, 'debian/tests/control')

    stanzas = read_stanzas(control_af)

    for stz in stanzas:
        try:
            try:
                tnames = stz['Tests']
            except KeyError:
                tnames = ['*']
                raise Unsupported(stz[' lno'],
                                  'no Tests field')
            tnames = map((lambda lt: lt[1]), tnames)
            tnames = string.join(tnames).split()
            base = {
                'restriction_names': [],
                'restrictions': [],
                'feature_names': [],
                'features': [],
                'testsdir': 'debian/tests',
                'depends': '@'
            }
            for fname in stz.keys():
                if fname.startswith(' '):
                    continue
                vl = stz[fname]
                try:
                    fclass = globals()['Field_' +
                                       fname.replace('-', '_')]
                except KeyError:
                    raise Unsupported(vl[0][0],
                                      'unknown metadata field %s' % fname)
                f = fclass(stz, fname, base, tnames, vl)
                f.parse()
            for tname in tnames:
                t = Test(tname, base, act)
                stz[' tests'].append(t)
        except Unsupported, u:
            for tname in tnames:
                u.report(tname)
            continue

    return stanzas


def print_exception(ei, msgprefix=''):
    if msgprefix:
        pstderr(msgprefix)
    (et, q, tb) = ei
    if et is Quit:
        pstderr('adt-run: ' + q.m)
        psummary('quitting: ' + q.m)
        return q.ec
    else:
        pstderr("adt-run: unexpected, exceptional, error:")
        psummary('quitting: unexpected error, consult transcript')
        traceback.print_exc(None, sys.stderr)
        if trace_stream is not None:
            traceback.print_exc(None, trace_stream)
        return 20


def cleanup():
    global trace_stream
    try:
        if testbed is not None:
            testbed.reset_apt()
            testbed.stop()
        if opts.tmp is None and tmp is not None:
            rmtree('tmp', tmp)
        if trace_stream is not None:
            trace_stream.close()
            trace_stream = None
    except:
        print_exception(sys.exc_info(),
                        '\nadt-run: error cleaning up:\n')
        os._exit(20)

#---------- registration, installation etc. of .deb's: Binaries


def determine_package(act):
    cmd = 'dpkg-deb --info --'.split(' ') + [act.af.read(), 'control']
    (rc, output) = subprocess_cooked(cmd, stdout=subprocess.PIPE)
    if rc:
        badpkg('failed to parse binary package, code %d' % rc)
    pkg_re = re.compile('^\s*Package\s*:\s*([0-9a-z][-+.0-9a-z]*)\s*$')
    act.pkg = None
    for l in output.split('\n'):
        m = pkg_re.match(l)
        if not m:
            continue
        if act.pkg:
            badpkg('two Package: lines in control file')
        act.pkg = m.groups()[0]
    if not act.pkg:
        badpkg('no good Package: line in control file')


def packages_from_source(act, tree):
    (rc, output) = subprocess_cooked(['dh_listpackages'],
                                     stdout=subprocess.PIPE, cwd=tree.read())
    if rc:
        badpkg('failed to parse packages built from source, code %d' % rc)

    # filter out empty lines
    packages = [p for p in output.split() if p]

    # filter out udebs
    control_af = RelativeInputFile(act.what + '-control',
                                   tree, 'debian/control')
    for st in read_stanzas(control_af):
        if 'Package' not in st:
                    # source stanza
            continue
        if 'Xc-package-type' in st:
            try:
                packages.remove(st['Package'][0][1])
            except ValueError:
                pass

    return packages


class Binaries:

    def __init__(self, tb):
        self.dir = TemporaryDir('binaries')
        self.dir.write()
        ok = False

        if opts.gnupghome is None:
            opts.gnupghome = tmp + '/gnupg'

        self._debug('initialising')
        try:
            for x in ['pubring', 'secring']:
                os.stat(opts.gnupghome + '/' + x + '.gpg')
            ok = True
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise

        if ok:
            self._debug('no key generation needed')
        else:
            self.genkey()

    def _debug(self, s):
        debug('* ' + s, 1)

    def genkey(self):
        self._debug('preparing for key generation')

        mkdir_okexist(os.path.dirname(opts.gnupghome), 02755)
        mkdir_okexist(opts.gnupghome, 0700)

        script = '''
  exec >&2
  cd "$1"
  cat <<"END" >key-gen-params
Key-Type: DSA
Key-Length: 1024
Key-Usage: sign
Name-Real: autopkgtest per-run key
Name-Comment: do not trust this key
Name-Email: autopkgtest@example.com
END
  set -x
  gpg --homedir="$1" --batch --gen-key key-gen-params
'''
        cmdl = ['sh', '-ec', script, 'x', opts.gnupghome]
        rc = subprocess_cooked(cmdl, dbg=('genkey', script))[0]
        if rc:
            bomb('key generation failed, code %d' % rc)

    def apt_configs(self):
        return {
            "Debug::pkgProblemResolver": "true",
            "APT::Get::force-yes": "true",
            "APT::Get::Assume-Yes": "true",
            "quiet": "true",
        }

    def _configure_apt(self, tb):
        config = OutputFile('apt-config', '/etc/apt/apt.conf.d/90autopkgtest',
                            True)
        f = open(config.write(), 'w')
        for (k, v) in self.apt_configs().iteritems():
            print >>f, '%s { "%s"; };' % (k, v)
        f.close()
        config.read(True)

        prefs = OutputFile('apt-prefs', '/etc/apt/preferences.d/90autopkgtest',
                           True)
        print >>open(prefs.write(), 'w'), '''
Package: *
Pin: origin ""
Pin-Priority: 1002
'''
        prefs.read(True)

    def _apt_get(self):
        ag = ['apt-get', '-q']
        for kv in self.apt_configs().iteritems():
            ag += ['-o', '%s=%s' % kv]
        return ' '.join(ag)

    def reset(self):
        self._debug('reset')
        rmtree('binaries', self.dir.read())
        self.dir.invalidate()
        self.dir.write()
        self.install = []
        self.blamed = []
        self.registered = set()

    def register(self, act, pkg, af, forwhat, blamed):
        self._debug('register what=%s deb_%s=%s pkg=%s af=%s' %
                    (act.what, forwhat, act.ah['deb_' + forwhat], pkg, str(af)))

        if act.ah['deb_' + forwhat] == 'ignore':
            return

        self.blamed += testbed.blamed

        leafname = pkg + '.deb'
        dest = RelativeOutputFile('binaries--' + leafname, self.dir, leafname)

        try:
            os.remove(dest.write())
        except (IOError, OSError), oe:
            if oe.errno != errno.ENOENT:
                raise oe

        try:
            os.link(af.read(), dest.write())
        except (IOError, OSError), oe:
            if oe.errno != errno.EXDEV:
                raise oe
            shutil.copy(af.read(), dest.write())

        if act.ah['deb_' + forwhat] == 'install':
            self.install.append(pkg)

        self.registered.add(pkg)

    def publish(self):
        self._debug('publish')
        if not self.registered:
            self._debug('no registered binaries, not publishing anything')
            return

        self._configure_apt(testbed)

        script = '''
  exec >&2
  cd "$1"
  apt-ftparchive packages . >Packages
  gzip <Packages >Packages.gz
  apt-ftparchive release . >Release
  rm -f Release.gpg
  gpg --homedir="$2" --batch --detach-sign --armour -o Release.gpg Release
  gpg --homedir="$2" --batch --export >archive-key.pgp
'''
        cmdl = ['sh', '-ec', script, 'x', self.dir.write(), opts.gnupghome]
        rc = subprocess_cooked(cmdl, dbg=('ftparchive', script))[0]
        if rc:
            bomb('apt-ftparchive or signature failed, code %d' % rc)

        self.dir.invalidate(True)
        apt_source = self.dir.read(True)

        so = TemporaryFile('vlds')
        script = '''
  exec 3>&1 >&2
  apt-key add archive-key.pgp
  echo "deb file://''' + apt_source + ''' /" >/etc/apt/sources.list.d/autopkgtest.list
  if [ "x`ls /var/lib/dpkg/updates`" != x ]; then
    echo >&2 "/var/lib/dpkg/updates contains some files, aargh"; exit 1
  fi
''' + self._apt_get() + ''' update >&2
  cat /var/lib/dpkg/status >&3
'''
        testbed.mungeing_apt()
        rc = testbed.execute('apt-key', ['sh', '-ec', script],
                             so=so.write(True), cwd=self.dir.write(True),
                             script=script, kind='install')
        if rc:
            bomb('apt setup failed with exit code %d' % rc)

        testbed.blamed += self.blamed

        self._debug('publish reinstall checking...')
        pkgs_reinstall = set()
        pkg = None
        for l in open(so.read()):
            if l.startswith('Package: '):
                pkg = l[9:].rstrip()
            elif l.startswith('Status: install '):
                if pkg in self.registered:
                    pkgs_reinstall.add(pkg)
                    self._debug(' publish reinstall needs ' + pkg)

        if pkgs_reinstall:
            for pkg in pkgs_reinstall:
                testbed.blame(pkg)
            what = 'apt-get-reinstall'
            cmdl = (self._apt_get() + ' --reinstall install ' +
                    ' '.join([pkg for pkg in pkgs_reinstall]) + ' >&2')
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg("installation of basic binarries failed,"
                       " exit code %d" % rc)

        self._debug('publish install...')
        for pkg in self.install:
            what = 'apt-get-install-%s' % pkg
            testbed.blame(pkg)
            cmdl = self._apt_get() + ' install ' + pkg + ' >&2'
            cmdl = ['sh', '-c', cmdl]
            rc = testbed.execute(what, cmdl, script=None, kind='install')
            if rc:
                badpkg("installation of %s failed, exit code %d"
                       % (pkg, rc))

        self._debug('publish done')

#---------- processing of sources (building)


def source_rules_command(act, script, what, which, work, cwd,
                         results_lines=0, xargs=[]):
    if opts.debuglevel >= 1:
        script = ["exec 3>&1 >&2", "set -x"] + script
    else:
        script = ["exec 3>&1 >&2"] + script
    script = '\n'.join(script)
    so = TemporaryFile('%s-%s-results' % (what, which))
    rc = testbed.execute('%s-%s' % (what, which),
                         ['sh', '-ec', script] + xargs, script=script,
                         so=so.write(True), cwd=cwd, kind='build')
    results = open(so.read()).read().rstrip('\n')
    if len(results):
        results = results.split("\n")
    else:
        results = []
    if rc:
        badpkg("rules %s failed with exit code %d" % (which, rc))
    if results_lines is not None and len(results) != results_lines:
        badpkg("got %d lines of results from %s where %d expected"
               % (len(results), which, results_lines))
    if results_lines == 1:
        return results[0]
    return results


def build_source(act, control_override):
    act.blame = 'arg:' + act.af.spec
    testbed.blame(act.blame)
    testbed.prepare1([])
    testbed.needs_reset()

    what = act.what
    debiancontrol = None
    act.binaries = []

    def debug_b(m):
        debug('* <%s:%s> %s' % (act.kind, act.what, m))

    if act.kind == 'dsc':
        dsc = act.af
        dsc_file = open(dsc.read())
        in_files = False
        fre = re.compile('^\s+[0-9a-f]+\s+\d+\s+([^/.][^/]*)$')
        for l in dsc_file:
            l = l.rstrip('\n')
            if l.startswith('Files:'):
                in_files = True
                continue
            elif l.startswith('#'):
                pass
            elif not l.startswith(' '):
                in_files = False
                if l.startswith('Source:'):
                    act.blame = 'dsc:' + l[7:].strip()
                    testbed.blame(act.blame)
            if not in_files:
                continue

            m = fre.match(l)
            if not m:
                badpkg(".dsc contains unparseable line"
                       " in Files: `%s'" % l)
            leaf = m.groups(0)[0]
            subfile = RelativeInputFile(what + '/' + leaf, dsc, leaf,
                                        sibling=True)
            subfile.read(True)
        dsc.read(True)

    if act.kind == 'ubtree':
        debiancontrol = RelativeInputFile(what + '-debiancontrol',
                                          act.af, 'debian/control')
        dsc = TemporaryFile(what + '-fakedsc')
        dsc_w = open(dsc.write(), 'w')
        for l in open(debiancontrol.read()):
            l = l.rstrip('\n')
            if not len(l):
                break
            print >>dsc_w, l
        print >>dsc_w, 'Binary: none-so-this-is-not-a-package-name'
        dsc_w.close()

    if act.kind == 'dsc':
        testbed.prepare2([])
        testbed.satisfy_dependencies_string('dpkg-dev',
                                            'install dpkg-dev')

    work = TemporaryDir(what + '-build')
    act.work = work

    tmpdir = work.write(True) + '/tmpdir'
    tmpdir_script = [
        'TMPDIR="$1"',
        'rm -rf -- "$TMPDIR"',
        'export TMPDIR',
        opts.user_wrap('mkdir -m 1777 -- "$TMPDIR"'),
    ]

    if act.kind == 'ubtree':
        spec = '%s/real-tree' % work.write(True)
        create_command = '''
    rm -rf "$spec"
    mkdir "$spec"
    cp -rP --preserve=timestamps,links -- "$origpwd"/. "$spec"/.
'''
        initcwd = act.af.read(True)

    if act.kind == 'dsc':
        spec = dsc.read(True)
        create_command = 'dpkg-source -x $spec\n'
        initcwd = work.write(True)

    script = [
        'spec="$2"',
        'origpwd=`pwd`',
        'cd ' + work.write(True)
    ]

    if opts.user:
        script += (['chown ' + opts.user + ' . ..'] +
                   tmpdir_script +
                   ['spec="$spec" origpwd="$origpwd" '
                    + opts.user_wrap(create_command)])
    else:
        script += (tmpdir_script +
                   [create_command])

    script += [
        'cd [a-z0-9]*-*/.',
        'pwd >&3',
        'set +e; test -f debian/tests/control; echo $? >&3'
    ]
    (result_pwd, control_test_rc) = source_rules_command(
        act, script, what, 'extract', work,
        cwd=initcwd, results_lines=2, xargs=['x', tmpdir, spec])

    filter = act.ah['dsc_filter']

    if control_test_rc == '1':
        act.missing_tests_control = True

    # For optional builds:
    #
    # We might need to build the package because:
    #   - we want its binaries (filter isn't _ and at least one of the
    #	deb_... isn't ignore)
    #   - the test control file says so
    #       (assuming we have any tests)

    class NeedBuildException:
        pass

    def build_needed(m):
        debug_b('build needed for %s' % m)
        raise NeedBuildException()

    try:
        if filter != '_' and (act.ah['deb_forbuilds'] != 'ignore' or
                              act.ah['deb_fortests'] != 'ignore'):
            build_needed('binaries')

        result_pwd_af = InputDir(what + '-treeforcontrol',
                                 result_pwd, True)
        stanzas = read_control(act, result_pwd_af, control_override)
        for stanza in stanzas:
            for t in stanza[' tests']:
                if 'build-needed' in t.restriction_names:
                    build_needed('test %s' % t.tname)

        debug_b('build not needed')
        built = False

    except NeedBuildException:

        if act.kind != 'dsc':
            testbed.prepare2([])

        testbed.satisfy_dependencies_string('build-essential',
                                            'install build-essential')
        testbed.satisfy_dependencies_dsc(dsc, 'build dependencies')

        script = tmpdir_script + [
            'cd "$2"',
            'dpkg-checkbuilddeps',
            opts.user_wrap('debian/rules build'),
        ]
        source_rules_command(act, script, what, 'build', work,
                             cwd=initcwd, xargs=['x', tmpdir, result_pwd])

        if os.path.dirname(result_pwd) + '/' != work.read(True):
            badpkg("results dir `%s' is not in expected parent"
                   " dir `%s'" % (result_pwd, work.read(True)))

        built = True

    act.tests_tree = RelativeInputDir(what + '-tests-tree',
                                      work, os.path.basename(result_pwd),
                                      True)

    if not built:
        act.blamed = []
        return

    act.blamed = copy.copy(testbed.blamed)

    debug_b('filter=%s' % filter)
    if filter != '_':
        script = tmpdir_script + [
            'cd ' + work.write(True) + '/[a-z0-9]*-*/.',
            opts.user_wrap(opts.gainroot + ' debian/rules binary'),
            'cd ..',
            'echo *.deb >&3',
        ]
        result_debs = source_rules_command(act, script, what,
                                           'binary', work, work.write(True),
                                           results_lines=1,
                                           xargs=['x', tmpdir])
        if result_debs == '*.deb':
            debs = []
        else:
            debs = result_debs.split(' ')
        debug_b('debs=' + repr(debs))
        deb_re = re.compile('^([-+.0-9a-z]+)_[^_/]+(?:_[^_/]+)\.deb$')
        for deb in debs:
            m = deb_re.match(deb)
            if not m:
                badpkg("badly-named binary `%s'" % deb)
            pkg = m.groups()[0]
            debug_b(' deb=%s, pkg=%s' % (deb, pkg))
            for pat in filter.split(','):
                debug_b('  pat=%s' % pat)
                if not fnmatch.fnmatchcase(pkg, pat):
                    debug_b('   no match')
                    continue
                deb_what = pkg + '_' + what + '.deb'
                bin = RelativeInputFile(deb_what, work, deb, True)
                debug_b('  deb_what=%s, bin=%s' %
                        (deb_what, str(bin)))
                binaries.register(act, pkg, bin,
                                  'forbuilds', testbed.blamed)
                act.binaries.append((pkg, bin))
                break
        debug_b('all done.')

#---------- main processing loop and main program


def process_actions():
    global binaries

    def debug_a1(m):
        debug('@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a2(m):
        debug('@@@@@@@@@@@@@@@@@@@@ ' + m)

    def debug_a3(m):
        debug('@@@@@@@@@@ ' + m)

    testbed.open()
    binaries = Binaries(testbed)

    binaries.reset()
    control_override = None

    debug_a1('builds ...')
    for act in opts.actions:
        debug_a2('%s %s' % (act.kind, act.what))

        if act.kind == 'control':
            control_override = act.af
        if act.kind == 'deb':
            testbed.blame('arg:' + act.af.spec)
            determine_package(act)
            testbed.blame('deb:' + act.pkg)
            binaries.register(act, act.pkg, act.af,
                              'forbuilds', testbed.blamed)
        if act.kind == 'dsc' or act.kind == 'ubtree':
            build_source(act, control_override)
        if act.kind == 'tree':
            act.binaries = []
        if act.kind.endswith('tree') or act.kind == 'dsc':
            control_override = None
        if act.kind == 'instantiate':
            pass

    debug_a1('builds done.')

    binaries.reset()
    control_override = None

    debug_a1('tests ...')
    for act in opts.actions:
        debug_a2('test %s %s' % (act.kind, act.what))

        testbed.needs_reset()
        if act.kind == 'control':
            control_override = act.af
        if act.kind == 'deb':
            binaries.register(act, act.pkg, act.af, 'fortests',
                              ['deb:' + act.pkg])
        if act.kind == 'dsc' or act.kind == 'ubtree':
            for (pkg, bin) in act.binaries:
                binaries.register(act, pkg, bin, 'fortests',
                                  act.blamed)
        if act.kind == 'dsc':
            if act.ah['dsc_tests']:
                debug_a3('read control ...')
                stanzas = read_control(act, act.tests_tree,
                                       control_override)
                testbed.blamed += act.blamed
                debug_a3('run_tests ...')
                run_tests(stanzas, act.tests_tree)
            control_override = None
        if act.kind == 'tree' or act.kind == 'ubtree':
            testbed.blame('arg:' + act.af.spec)
            stanzas = read_control(act, act.af, control_override)
            debug_a3('run_tests ...')
            if act.kind == 'ubtree':
                run_tests(stanzas, act.tests_tree)
            else:
                run_tests(stanzas, act.af)
            control_override = None
        if act.kind == 'instantiate':
            testbed.prepare([])
    debug_a1('tests done.')


def main():
    global testbed
    global tmp
    try:
        parse_args()
    except SystemExit:
        os._exit(20)
    try:
        setup_trace()
        testbed = Testbed()
        testbed.start()
        finalise_options()
        process_actions()
    except:
        ec = print_exception(sys.exc_info(), '')
        cleanup()
        os._exit(ec)
    cleanup()
    os._exit(errorcode)

main()
