Subversion Repositories openrisc

# -*- python -*-

## Copyright (C) 2005, 2006, 2008 Free Software Foundation

## Written by Gary Benson 

##

## This program is free software; you can redistribute it and/or modify

## it under the terms of the GNU General Public License as published by

## the Free Software Foundation; either version 2 of the License, or

## (at your option) any later version.

##

## This program is distributed in the hope that it will be useful,

## but WITHOUT ANY WARRANTY; without even the implied warranty of

## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

## GNU General Public License for more details.

import classfile

import copy

# The md5 module is deprecated in Python 2.5

try:

    from hashlib import md5

except ImportError:

    from md5 import md5

import operator

import os

import sys

import cStringIO as StringIO

import zipfile

PATHS = {"make":   "@MAKE@",

         "gcj":    "@prefix@/bin/gcj@gcc_suffix@",

         "dbtool": "@prefix@/bin/gcj-dbtool@gcc_suffix@"}

MAKEFLAGS = []

GCJFLAGS = ["-fPIC", "-findirect-dispatch", "-fjni"]

LDFLAGS = ["-Wl,-Bsymbolic"]

MAX_CLASSES_PER_JAR = 1024

MAX_BYTES_PER_JAR = 1048576

MAKEFILE = "Makefile"

MAKEFILE_HEADER = '''\

GCJ = %(gcj)s

DBTOOL = %(dbtool)s

GCJFLAGS = %(gcjflags)s

LDFLAGS = %(ldflags)s

%%.o: %%.jar

        $(GCJ) -c $(GCJFLAGS) $< -o $@

TARGETS = \\

%(targets)s

all: $(TARGETS)'''

MAKEFILE_JOB = '''

%(base)s_SOURCES = \\

%(jars)s

%(base)s_OBJECTS = \\

$(%(base)s_SOURCES:.jar=.o)

%(dso)s: $(%(base)s_OBJECTS)

        $(GCJ) -shared $(GCJFLAGS) $(LDFLAGS) $^ -o $@

%(db)s: $(%(base)s_SOURCES)

        $(DBTOOL) -n $@ 64

        for jar in $^; do \\

            $(DBTOOL) -f $@ $$jar \\

                %(libdir)s/%(dso)s; \\

        done'''

ZIPMAGIC, CLASSMAGIC = "PK\x03\x04", "\xca\xfe\xba\xbe"

class Error(Exception):

    pass

class Compiler:

    def __init__(self, srcdir, libdir, prefix = None):

        self.srcdir = os.path.abspath(srcdir)

        self.libdir = os.path.abspath(libdir)

        if prefix is None:

            self.dstdir = self.libdir

        else:

            self.dstdir = os.path.join(prefix, self.libdir.lstrip(os.sep))

        # Calling code may modify these parameters

        self.gcjflags = copy.copy(GCJFLAGS)

        self.ldflags = copy.copy(LDFLAGS)

        self.makeflags = copy.copy(MAKEFLAGS)

        self.exclusions = []

    def compile(self):

        """Search srcdir for classes and jarfiles, then generate

        solibs and mappings databases for them all in libdir."""

        if not os.path.isdir(self.dstdir):

            os.makedirs(self.dstdir)

        oldcwd = os.getcwd()

        os.chdir(self.dstdir)

        try:

            jobs = self.getJobList()

            if not jobs:

                raise Error, "nothing to do"

            self.writeMakefile(MAKEFILE, jobs)

            for job in jobs:

                job.writeJars()

            system([PATHS["make"]] + self.makeflags)

            for job in jobs:

                job.clean()

            os.unlink(MAKEFILE)

        finally:

            os.chdir(oldcwd)

    def getJobList(self):

        """Return all jarfiles and class collections in srcdir."""

        jobs = weed_jobs(find_jobs(self.srcdir, self.exclusions))

        set_basenames(jobs)

        return jobs

    def writeMakefile(self, path, jobs):

        """Generate a makefile to build the solibs and mappings

        databases for the specified list of jobs."""

        fp = open(path, "w")

        print >>fp, MAKEFILE_HEADER % {

            "gcj": PATHS["gcj"],

            "dbtool": PATHS["dbtool"],

            "gcjflags": " ".join(self.gcjflags),

            "ldflags": " ".join(self.ldflags),

            "targets": " \\\n".join(reduce(operator.add, [

                (job.dsoName(), job.dbName()) for job in jobs]))}

        for job in jobs:

            values = job.ruleArguments()

            values["libdir"] = self.libdir

            print >>fp, MAKEFILE_JOB % values

        fp.close()

def find_jobs(dir, exclusions = ()):

    """Scan a directory and find things to compile: jarfiles (zips,

    wars, ears, rars, etc: we go by magic rather than file extension)

    and directories of classes."""

    def visit((classes, zips), dir, items):

        for item in items:

            path = os.path.join(dir, item)

            if os.path.islink(path) or not os.path.isfile(path):

                continue

            magic = open(path, "r").read(4)

            if magic == ZIPMAGIC:

                zips.append(path)

            elif magic == CLASSMAGIC:

                classes.append(path)

    classes, paths = [], []

    os.path.walk(dir, visit, (classes, paths))

    # Convert the list of classes into a list of directories

    while classes:

        # XXX this requires the class to be correctly located in its heirachy.

        path = classes[0][:-len(os.sep + classname(classes[0]) + ".class")]

        paths.append(path)

        classes = [cls for cls in classes if not cls.startswith(path)]

    # Handle exclusions.  We're really strict about them because the

    # option is temporary in aot-compile-rpm and dead options left in

    # specfiles will hinder its removal.

    for path in exclusions:

        if path in paths:

            paths.remove(path)

        else:

            raise Error, "%s: path does not exist or is not a job" % path

    # Build the list of jobs

    jobs = []

    paths.sort()

    for path in paths:

        if os.path.isfile(path):

            job = JarJob(path)

        else:

            job = DirJob(path)

        if len(job.classes):

            jobs.append(job)

    return jobs

class Job:

    """A collection of classes that will be compiled as a unit."""

    def __init__(self, path):

        self.path, self.classes, self.blocks = path, {}, None

        self.classnames = {}

    def addClass(self, bytes, name):

        """Subclasses call this from their __init__ method for

        every class they find."""

        digest = md5(bytes).digest()

        self.classes[digest] = bytes

        self.classnames[digest] = name

    def __makeBlocks(self):

        """Split self.classes into chunks that can be compiled to

        native code by gcj.  In the majority of cases this is not

        necessary -- the job will have come from a jarfile which will

        be equivalent to the one we generate -- but this only happens

        _if_ the job was a jarfile and _if_ the jarfile isn't too big

        and _if_ the jarfile has the correct extension and _if_ all

        classes are correctly named and _if_ the jarfile has no

        embedded jarfiles.  Fitting a special case around all these

        conditions is tricky to say the least.

        Note that this could be called at the end of each subclass's

        __init__ method.  The reason this is not done is because we

        need to parse every class file.  This is slow, and unnecessary

        if the job is subsetted."""

        names = {}

        for hash, bytes in self.classes.items():

            try:

                name = classname(bytes)

            except:

                warn("job %s: class %s malformed or not a valid class file" \

                     % (self.path, self.classnames[hash]))

                raise

            if not names.has_key(name):

                names[name] = []

            names[name].append(hash)

        names = names.items()

        # We have to sort somehow, or the jars we generate

        # We sort by name in a simplistic attempt to keep related

        # classes together so inter-class optimisation can happen.

        names.sort()

        self.blocks, bytes = [[]], 0

        for name, hashes in names:

            for hash in hashes:

                if len(self.blocks[-1]) >= MAX_CLASSES_PER_JAR \

                   or bytes >= MAX_BYTES_PER_JAR:

                    self.blocks.append([])

                    bytes = 0

                self.blocks[-1].append((name, hash))

                bytes += len(self.classes[hash])

    # From Archit Shah:

    #   The implementation and the documentation don't seem to match.

    #

    #    [a, b].isSubsetOf([a]) => True

    #

    #   Identical copies of all classes this collection do not exist

    #   in the other. I think the method should be named isSupersetOf

    #   and the documentation should swap uses of "this" and "other"

    #

    # XXX think about this when I've had more sleep...

    def isSubsetOf(self, other):

        """Returns True if identical copies of all classes in this

        collection exist in the other."""

        for item in other.classes.keys():

            if not self.classes.has_key(item):

                return False

        return True

    def __targetName(self, ext):

        return self.basename + ext

    def tempJarName(self, num):

        return self.__targetName(".%d.jar" % (num + 1))

    def tempObjName(self, num):

        return self.__targetName(".%d.o" % (num + 1))

    def dsoName(self):

        """Return the filename of the shared library that will be

        built from this job."""

        return self.__targetName(".so")

    def dbName(self):

        """Return the filename of the mapping database that will be

        built from this job."""

        return self.__targetName(".db")

    def ruleArguments(self):

        """Return a dictionary of values that when substituted

        into MAKEFILE_JOB will create the rules required to build

        the shared library and mapping database for this job."""

        if self.blocks is None:

            self.__makeBlocks()

        return {

            "base": "".join(

                [c.isalnum() and c or "_" for c in self.dsoName()]),

            "jars": " \\\n".join(

                [self.tempJarName(i) for i in xrange(len(self.blocks))]),

            "dso": self.dsoName(),

            "db": self.dbName()}

    def writeJars(self):

        """Generate jarfiles that can be native compiled by gcj."""

        if self.blocks is None:

            self.__makeBlocks()

        for block, i in zip(self.blocks, xrange(len(self.blocks))):

            jar = zipfile.ZipFile(self.tempJarName(i), "w", zipfile.ZIP_STORED)

            for name, hash in block:

                jar.writestr(

                    zipfile.ZipInfo("%s.class" % name), self.classes[hash])

            jar.close()

    def clean(self):

        """Delete all temporary files created during this job's build."""

        if self.blocks is None:

            self.__makeBlocks()

        for i in xrange(len(self.blocks)):

            os.unlink(self.tempJarName(i))

            os.unlink(self.tempObjName(i))

class JarJob(Job):

    """A Job whose origin was a jarfile."""

    def __init__(self, path):

        Job.__init__(self, path)

        self._walk(zipfile.ZipFile(path, "r"))

    def _walk(self, zf):

        for name in zf.namelist():

            bytes = zf.read(name)

            if bytes.startswith(ZIPMAGIC):

                self._walk(zipfile.ZipFile(StringIO.StringIO(bytes)))

            elif bytes.startswith(CLASSMAGIC):

                self.addClass(bytes, name)

class DirJob(Job):

    """A Job whose origin was a directory of classfiles."""

    def __init__(self, path):

        Job.__init__(self, path)

        os.path.walk(path, DirJob._visit, self)

    def _visit(self, dir, items):

        for item in items:

            path = os.path.join(dir, item)

            if os.path.islink(path) or not os.path.isfile(path):

                continue

            fp = open(path, "r")

            magic = fp.read(4)

            if magic == CLASSMAGIC:

                self.addClass(magic + fp.read(), name)

def weed_jobs(jobs):

    """Remove any jarfiles that are completely contained within

    another.  This is more common than you'd think, and we only

    need one nativified copy of each class after all."""

    jobs = copy.copy(jobs)

    while True:

        for job1 in jobs:

            for job2 in jobs:

                if job1 is job2:

                    continue

                if job1.isSubsetOf(job2):

                    msg = "subsetted %s" % job2.path

                    if job2.isSubsetOf(job1):

                        if (isinstance(job1, DirJob) and

                            isinstance(job2, JarJob)):

                            # In the braindead case where a package

                            # contains an expanded copy of a jarfile

                            # the jarfile takes precedence.

                            continue

                        msg += " (identical)"

                    warn(msg)

                    jobs.remove(job2)

                    break

            else:

                continue

            break

        else:

            break

        continue

    return jobs

def set_basenames(jobs):

    """Ensure that each jarfile has a different basename."""

    names = {}

    for job in jobs:

        name = os.path.basename(job.path)

        if not names.has_key(name):

            names[name] = []

        names[name].append(job)

    for name, set in names.items():

        if len(set) == 1:

            set[0].basename = name

            continue

        # prefix the jar filenames to make them unique

        # XXX will not work in most cases -- needs generalising

        set = [(job.path.split(os.sep), job) for job in set]

        minlen = min([len(bits) for bits, job in set])

        set = [(bits[-minlen:], job) for bits, job in set]

        bits = apply(zip, [bits for bits, job in set])

        while True:

            row = bits[-2]

            for bit in row[1:]:

                if bit != row[0]:

                    break

            else:

                del bits[-2]

                continue

            break

        set = zip(

            ["_".join(name) for name in apply(zip, bits[-2:])],

            [job for bits, job in set])

        for name, job in set:

            warn("building %s as %s" % (job.path, name))

            job.basename = name

    # XXX keep this check until we're properly general

    names = {}

    for job in jobs:

        name = job.basename

        if names.has_key(name):

            raise Error, "%s: duplicate jobname" % name

        names[name] = 1

def system(command):

    """Execute a command."""

    status = os.spawnv(os.P_WAIT, command[0], command)

    if status > 0:

        raise Error, "%s exited with code %d" % (command[0], status)

    elif status < 0:

        raise Error, "%s killed by signal %d" % (command[0], -status)

def warn(msg):

    """Print a warning message."""

    print >>sys.stderr, "%s: warning: %s" % (

        os.path.basename(sys.argv[0]), msg)

def classname(bytes):

    """Extract the class name from the bytes of a class file."""

    klass = classfile.Class(bytes)

    return klass.constants[klass.constants[klass.name][1]][1]