1 |
754 |
jeremybenn |
# -*- python -*-
|
2 |
|
|
|
3 |
|
|
## Copyright (C) 2005, 2006, 2008 Free Software Foundation
|
4 |
|
|
## Written by Gary Benson
|
5 |
|
|
##
|
6 |
|
|
## This program is free software; you can redistribute it and/or modify
|
7 |
|
|
## it under the terms of the GNU General Public License as published by
|
8 |
|
|
## the Free Software Foundation; either version 2 of the License, or
|
9 |
|
|
## (at your option) any later version.
|
10 |
|
|
##
|
11 |
|
|
## This program is distributed in the hope that it will be useful,
|
12 |
|
|
## but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 |
|
|
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14 |
|
|
## GNU General Public License for more details.
|
15 |
|
|
|
16 |
|
|
import classfile
|
17 |
|
|
import copy
|
18 |
|
|
# The md5 module is deprecated in Python 2.5
|
19 |
|
|
try:
|
20 |
|
|
from hashlib import md5
|
21 |
|
|
except ImportError:
|
22 |
|
|
from md5 import md5
|
23 |
|
|
import operator
|
24 |
|
|
import os
|
25 |
|
|
import sys
|
26 |
|
|
import cStringIO as StringIO
|
27 |
|
|
import zipfile
|
28 |
|
|
|
29 |
|
|
PATHS = {"make": "@MAKE@",
|
30 |
|
|
"gcj": "@prefix@/bin/gcj@gcc_suffix@",
|
31 |
|
|
"dbtool": "@prefix@/bin/gcj-dbtool@gcc_suffix@"}
|
32 |
|
|
|
33 |
|
|
MAKEFLAGS = []
|
34 |
|
|
GCJFLAGS = ["-fPIC", "-findirect-dispatch", "-fjni"]
|
35 |
|
|
LDFLAGS = ["-Wl,-Bsymbolic"]
|
36 |
|
|
|
37 |
|
|
MAX_CLASSES_PER_JAR = 1024
|
38 |
|
|
MAX_BYTES_PER_JAR = 1048576
|
39 |
|
|
|
40 |
|
|
MAKEFILE = "Makefile"
|
41 |
|
|
|
42 |
|
|
MAKEFILE_HEADER = '''\
|
43 |
|
|
GCJ = %(gcj)s
|
44 |
|
|
DBTOOL = %(dbtool)s
|
45 |
|
|
GCJFLAGS = %(gcjflags)s
|
46 |
|
|
LDFLAGS = %(ldflags)s
|
47 |
|
|
|
48 |
|
|
%%.o: %%.jar
|
49 |
|
|
$(GCJ) -c $(GCJFLAGS) $< -o $@
|
50 |
|
|
|
51 |
|
|
TARGETS = \\
|
52 |
|
|
%(targets)s
|
53 |
|
|
|
54 |
|
|
all: $(TARGETS)'''
|
55 |
|
|
|
56 |
|
|
MAKEFILE_JOB = '''
|
57 |
|
|
%(base)s_SOURCES = \\
|
58 |
|
|
%(jars)s
|
59 |
|
|
|
60 |
|
|
%(base)s_OBJECTS = \\
|
61 |
|
|
$(%(base)s_SOURCES:.jar=.o)
|
62 |
|
|
|
63 |
|
|
%(dso)s: $(%(base)s_OBJECTS)
|
64 |
|
|
$(GCJ) -shared $(GCJFLAGS) $(LDFLAGS) $^ -o $@
|
65 |
|
|
|
66 |
|
|
%(db)s: $(%(base)s_SOURCES)
|
67 |
|
|
$(DBTOOL) -n $@ 64
|
68 |
|
|
for jar in $^; do \\
|
69 |
|
|
$(DBTOOL) -f $@ $$jar \\
|
70 |
|
|
%(libdir)s/%(dso)s; \\
|
71 |
|
|
done'''
|
72 |
|
|
|
73 |
|
|
ZIPMAGIC, CLASSMAGIC = "PK\x03\x04", "\xca\xfe\xba\xbe"
|
74 |
|
|
|
75 |
|
|
class Error(Exception):
|
76 |
|
|
pass
|
77 |
|
|
|
78 |
|
|
class Compiler:
|
79 |
|
|
def __init__(self, srcdir, libdir, prefix = None):
|
80 |
|
|
self.srcdir = os.path.abspath(srcdir)
|
81 |
|
|
self.libdir = os.path.abspath(libdir)
|
82 |
|
|
if prefix is None:
|
83 |
|
|
self.dstdir = self.libdir
|
84 |
|
|
else:
|
85 |
|
|
self.dstdir = os.path.join(prefix, self.libdir.lstrip(os.sep))
|
86 |
|
|
|
87 |
|
|
# Calling code may modify these parameters
|
88 |
|
|
self.gcjflags = copy.copy(GCJFLAGS)
|
89 |
|
|
self.ldflags = copy.copy(LDFLAGS)
|
90 |
|
|
self.makeflags = copy.copy(MAKEFLAGS)
|
91 |
|
|
self.exclusions = []
|
92 |
|
|
|
93 |
|
|
def compile(self):
|
94 |
|
|
"""Search srcdir for classes and jarfiles, then generate
|
95 |
|
|
solibs and mappings databases for them all in libdir."""
|
96 |
|
|
if not os.path.isdir(self.dstdir):
|
97 |
|
|
os.makedirs(self.dstdir)
|
98 |
|
|
oldcwd = os.getcwd()
|
99 |
|
|
os.chdir(self.dstdir)
|
100 |
|
|
try:
|
101 |
|
|
jobs = self.getJobList()
|
102 |
|
|
if not jobs:
|
103 |
|
|
raise Error, "nothing to do"
|
104 |
|
|
self.writeMakefile(MAKEFILE, jobs)
|
105 |
|
|
for job in jobs:
|
106 |
|
|
job.writeJars()
|
107 |
|
|
system([PATHS["make"]] + self.makeflags)
|
108 |
|
|
for job in jobs:
|
109 |
|
|
job.clean()
|
110 |
|
|
os.unlink(MAKEFILE)
|
111 |
|
|
finally:
|
112 |
|
|
os.chdir(oldcwd)
|
113 |
|
|
|
114 |
|
|
def getJobList(self):
|
115 |
|
|
"""Return all jarfiles and class collections in srcdir."""
|
116 |
|
|
jobs = weed_jobs(find_jobs(self.srcdir, self.exclusions))
|
117 |
|
|
set_basenames(jobs)
|
118 |
|
|
return jobs
|
119 |
|
|
|
120 |
|
|
def writeMakefile(self, path, jobs):
|
121 |
|
|
"""Generate a makefile to build the solibs and mappings
|
122 |
|
|
databases for the specified list of jobs."""
|
123 |
|
|
fp = open(path, "w")
|
124 |
|
|
print >>fp, MAKEFILE_HEADER % {
|
125 |
|
|
"gcj": PATHS["gcj"],
|
126 |
|
|
"dbtool": PATHS["dbtool"],
|
127 |
|
|
"gcjflags": " ".join(self.gcjflags),
|
128 |
|
|
"ldflags": " ".join(self.ldflags),
|
129 |
|
|
"targets": " \\\n".join(reduce(operator.add, [
|
130 |
|
|
(job.dsoName(), job.dbName()) for job in jobs]))}
|
131 |
|
|
for job in jobs:
|
132 |
|
|
values = job.ruleArguments()
|
133 |
|
|
values["libdir"] = self.libdir
|
134 |
|
|
print >>fp, MAKEFILE_JOB % values
|
135 |
|
|
fp.close()
|
136 |
|
|
|
137 |
|
|
def find_jobs(dir, exclusions = ()):
|
138 |
|
|
"""Scan a directory and find things to compile: jarfiles (zips,
|
139 |
|
|
wars, ears, rars, etc: we go by magic rather than file extension)
|
140 |
|
|
and directories of classes."""
|
141 |
|
|
def visit((classes, zips), dir, items):
|
142 |
|
|
for item in items:
|
143 |
|
|
path = os.path.join(dir, item)
|
144 |
|
|
if os.path.islink(path) or not os.path.isfile(path):
|
145 |
|
|
continue
|
146 |
|
|
magic = open(path, "r").read(4)
|
147 |
|
|
if magic == ZIPMAGIC:
|
148 |
|
|
zips.append(path)
|
149 |
|
|
elif magic == CLASSMAGIC:
|
150 |
|
|
classes.append(path)
|
151 |
|
|
classes, paths = [], []
|
152 |
|
|
os.path.walk(dir, visit, (classes, paths))
|
153 |
|
|
# Convert the list of classes into a list of directories
|
154 |
|
|
while classes:
|
155 |
|
|
# XXX this requires the class to be correctly located in its heirachy.
|
156 |
|
|
path = classes[0][:-len(os.sep + classname(classes[0]) + ".class")]
|
157 |
|
|
paths.append(path)
|
158 |
|
|
classes = [cls for cls in classes if not cls.startswith(path)]
|
159 |
|
|
# Handle exclusions. We're really strict about them because the
|
160 |
|
|
# option is temporary in aot-compile-rpm and dead options left in
|
161 |
|
|
# specfiles will hinder its removal.
|
162 |
|
|
for path in exclusions:
|
163 |
|
|
if path in paths:
|
164 |
|
|
paths.remove(path)
|
165 |
|
|
else:
|
166 |
|
|
raise Error, "%s: path does not exist or is not a job" % path
|
167 |
|
|
# Build the list of jobs
|
168 |
|
|
jobs = []
|
169 |
|
|
paths.sort()
|
170 |
|
|
for path in paths:
|
171 |
|
|
if os.path.isfile(path):
|
172 |
|
|
job = JarJob(path)
|
173 |
|
|
else:
|
174 |
|
|
job = DirJob(path)
|
175 |
|
|
if len(job.classes):
|
176 |
|
|
jobs.append(job)
|
177 |
|
|
return jobs
|
178 |
|
|
|
179 |
|
|
class Job:
|
180 |
|
|
"""A collection of classes that will be compiled as a unit."""
|
181 |
|
|
|
182 |
|
|
def __init__(self, path):
|
183 |
|
|
self.path, self.classes, self.blocks = path, {}, None
|
184 |
|
|
self.classnames = {}
|
185 |
|
|
|
186 |
|
|
def addClass(self, bytes, name):
|
187 |
|
|
"""Subclasses call this from their __init__ method for
|
188 |
|
|
every class they find."""
|
189 |
|
|
digest = md5(bytes).digest()
|
190 |
|
|
self.classes[digest] = bytes
|
191 |
|
|
self.classnames[digest] = name
|
192 |
|
|
|
193 |
|
|
def __makeBlocks(self):
|
194 |
|
|
"""Split self.classes into chunks that can be compiled to
|
195 |
|
|
native code by gcj. In the majority of cases this is not
|
196 |
|
|
necessary -- the job will have come from a jarfile which will
|
197 |
|
|
be equivalent to the one we generate -- but this only happens
|
198 |
|
|
_if_ the job was a jarfile and _if_ the jarfile isn't too big
|
199 |
|
|
and _if_ the jarfile has the correct extension and _if_ all
|
200 |
|
|
classes are correctly named and _if_ the jarfile has no
|
201 |
|
|
embedded jarfiles. Fitting a special case around all these
|
202 |
|
|
conditions is tricky to say the least.
|
203 |
|
|
|
204 |
|
|
Note that this could be called at the end of each subclass's
|
205 |
|
|
__init__ method. The reason this is not done is because we
|
206 |
|
|
need to parse every class file. This is slow, and unnecessary
|
207 |
|
|
if the job is subsetted."""
|
208 |
|
|
names = {}
|
209 |
|
|
for hash, bytes in self.classes.items():
|
210 |
|
|
try:
|
211 |
|
|
name = classname(bytes)
|
212 |
|
|
except:
|
213 |
|
|
warn("job %s: class %s malformed or not a valid class file" \
|
214 |
|
|
% (self.path, self.classnames[hash]))
|
215 |
|
|
raise
|
216 |
|
|
if not names.has_key(name):
|
217 |
|
|
names[name] = []
|
218 |
|
|
names[name].append(hash)
|
219 |
|
|
names = names.items()
|
220 |
|
|
# We have to sort somehow, or the jars we generate
|
221 |
|
|
# We sort by name in a simplistic attempt to keep related
|
222 |
|
|
# classes together so inter-class optimisation can happen.
|
223 |
|
|
names.sort()
|
224 |
|
|
self.blocks, bytes = [[]], 0
|
225 |
|
|
for name, hashes in names:
|
226 |
|
|
for hash in hashes:
|
227 |
|
|
if len(self.blocks[-1]) >= MAX_CLASSES_PER_JAR \
|
228 |
|
|
or bytes >= MAX_BYTES_PER_JAR:
|
229 |
|
|
self.blocks.append([])
|
230 |
|
|
bytes = 0
|
231 |
|
|
self.blocks[-1].append((name, hash))
|
232 |
|
|
bytes += len(self.classes[hash])
|
233 |
|
|
|
234 |
|
|
# From Archit Shah:
|
235 |
|
|
# The implementation and the documentation don't seem to match.
|
236 |
|
|
#
|
237 |
|
|
# [a, b].isSubsetOf([a]) => True
|
238 |
|
|
#
|
239 |
|
|
# Identical copies of all classes this collection do not exist
|
240 |
|
|
# in the other. I think the method should be named isSupersetOf
|
241 |
|
|
# and the documentation should swap uses of "this" and "other"
|
242 |
|
|
#
|
243 |
|
|
# XXX think about this when I've had more sleep...
|
244 |
|
|
def isSubsetOf(self, other):
|
245 |
|
|
"""Returns True if identical copies of all classes in this
|
246 |
|
|
collection exist in the other."""
|
247 |
|
|
for item in other.classes.keys():
|
248 |
|
|
if not self.classes.has_key(item):
|
249 |
|
|
return False
|
250 |
|
|
return True
|
251 |
|
|
|
252 |
|
|
def __targetName(self, ext):
|
253 |
|
|
return self.basename + ext
|
254 |
|
|
|
255 |
|
|
def tempJarName(self, num):
|
256 |
|
|
return self.__targetName(".%d.jar" % (num + 1))
|
257 |
|
|
|
258 |
|
|
def tempObjName(self, num):
|
259 |
|
|
return self.__targetName(".%d.o" % (num + 1))
|
260 |
|
|
|
261 |
|
|
def dsoName(self):
|
262 |
|
|
"""Return the filename of the shared library that will be
|
263 |
|
|
built from this job."""
|
264 |
|
|
return self.__targetName(".so")
|
265 |
|
|
|
266 |
|
|
def dbName(self):
|
267 |
|
|
"""Return the filename of the mapping database that will be
|
268 |
|
|
built from this job."""
|
269 |
|
|
return self.__targetName(".db")
|
270 |
|
|
|
271 |
|
|
def ruleArguments(self):
|
272 |
|
|
"""Return a dictionary of values that when substituted
|
273 |
|
|
into MAKEFILE_JOB will create the rules required to build
|
274 |
|
|
the shared library and mapping database for this job."""
|
275 |
|
|
if self.blocks is None:
|
276 |
|
|
self.__makeBlocks()
|
277 |
|
|
return {
|
278 |
|
|
"base": "".join(
|
279 |
|
|
[c.isalnum() and c or "_" for c in self.dsoName()]),
|
280 |
|
|
"jars": " \\\n".join(
|
281 |
|
|
[self.tempJarName(i) for i in xrange(len(self.blocks))]),
|
282 |
|
|
"dso": self.dsoName(),
|
283 |
|
|
"db": self.dbName()}
|
284 |
|
|
|
285 |
|
|
def writeJars(self):
|
286 |
|
|
"""Generate jarfiles that can be native compiled by gcj."""
|
287 |
|
|
if self.blocks is None:
|
288 |
|
|
self.__makeBlocks()
|
289 |
|
|
for block, i in zip(self.blocks, xrange(len(self.blocks))):
|
290 |
|
|
jar = zipfile.ZipFile(self.tempJarName(i), "w", zipfile.ZIP_STORED)
|
291 |
|
|
for name, hash in block:
|
292 |
|
|
jar.writestr(
|
293 |
|
|
zipfile.ZipInfo("%s.class" % name), self.classes[hash])
|
294 |
|
|
jar.close()
|
295 |
|
|
|
296 |
|
|
def clean(self):
|
297 |
|
|
"""Delete all temporary files created during this job's build."""
|
298 |
|
|
if self.blocks is None:
|
299 |
|
|
self.__makeBlocks()
|
300 |
|
|
for i in xrange(len(self.blocks)):
|
301 |
|
|
os.unlink(self.tempJarName(i))
|
302 |
|
|
os.unlink(self.tempObjName(i))
|
303 |
|
|
|
304 |
|
|
class JarJob(Job):
|
305 |
|
|
"""A Job whose origin was a jarfile."""
|
306 |
|
|
|
307 |
|
|
def __init__(self, path):
|
308 |
|
|
Job.__init__(self, path)
|
309 |
|
|
self._walk(zipfile.ZipFile(path, "r"))
|
310 |
|
|
|
311 |
|
|
def _walk(self, zf):
|
312 |
|
|
for name in zf.namelist():
|
313 |
|
|
bytes = zf.read(name)
|
314 |
|
|
if bytes.startswith(ZIPMAGIC):
|
315 |
|
|
self._walk(zipfile.ZipFile(StringIO.StringIO(bytes)))
|
316 |
|
|
elif bytes.startswith(CLASSMAGIC):
|
317 |
|
|
self.addClass(bytes, name)
|
318 |
|
|
|
319 |
|
|
class DirJob(Job):
|
320 |
|
|
"""A Job whose origin was a directory of classfiles."""
|
321 |
|
|
|
322 |
|
|
def __init__(self, path):
|
323 |
|
|
Job.__init__(self, path)
|
324 |
|
|
os.path.walk(path, DirJob._visit, self)
|
325 |
|
|
|
326 |
|
|
def _visit(self, dir, items):
|
327 |
|
|
for item in items:
|
328 |
|
|
path = os.path.join(dir, item)
|
329 |
|
|
if os.path.islink(path) or not os.path.isfile(path):
|
330 |
|
|
continue
|
331 |
|
|
fp = open(path, "r")
|
332 |
|
|
magic = fp.read(4)
|
333 |
|
|
if magic == CLASSMAGIC:
|
334 |
|
|
self.addClass(magic + fp.read(), name)
|
335 |
|
|
|
336 |
|
|
def weed_jobs(jobs):
|
337 |
|
|
"""Remove any jarfiles that are completely contained within
|
338 |
|
|
another. This is more common than you'd think, and we only
|
339 |
|
|
need one nativified copy of each class after all."""
|
340 |
|
|
jobs = copy.copy(jobs)
|
341 |
|
|
while True:
|
342 |
|
|
for job1 in jobs:
|
343 |
|
|
for job2 in jobs:
|
344 |
|
|
if job1 is job2:
|
345 |
|
|
continue
|
346 |
|
|
if job1.isSubsetOf(job2):
|
347 |
|
|
msg = "subsetted %s" % job2.path
|
348 |
|
|
if job2.isSubsetOf(job1):
|
349 |
|
|
if (isinstance(job1, DirJob) and
|
350 |
|
|
isinstance(job2, JarJob)):
|
351 |
|
|
# In the braindead case where a package
|
352 |
|
|
# contains an expanded copy of a jarfile
|
353 |
|
|
# the jarfile takes precedence.
|
354 |
|
|
continue
|
355 |
|
|
msg += " (identical)"
|
356 |
|
|
warn(msg)
|
357 |
|
|
jobs.remove(job2)
|
358 |
|
|
break
|
359 |
|
|
else:
|
360 |
|
|
continue
|
361 |
|
|
break
|
362 |
|
|
else:
|
363 |
|
|
break
|
364 |
|
|
continue
|
365 |
|
|
return jobs
|
366 |
|
|
|
367 |
|
|
def set_basenames(jobs):
|
368 |
|
|
"""Ensure that each jarfile has a different basename."""
|
369 |
|
|
names = {}
|
370 |
|
|
for job in jobs:
|
371 |
|
|
name = os.path.basename(job.path)
|
372 |
|
|
if not names.has_key(name):
|
373 |
|
|
names[name] = []
|
374 |
|
|
names[name].append(job)
|
375 |
|
|
for name, set in names.items():
|
376 |
|
|
if len(set) == 1:
|
377 |
|
|
set[0].basename = name
|
378 |
|
|
continue
|
379 |
|
|
# prefix the jar filenames to make them unique
|
380 |
|
|
# XXX will not work in most cases -- needs generalising
|
381 |
|
|
set = [(job.path.split(os.sep), job) for job in set]
|
382 |
|
|
minlen = min([len(bits) for bits, job in set])
|
383 |
|
|
set = [(bits[-minlen:], job) for bits, job in set]
|
384 |
|
|
bits = apply(zip, [bits for bits, job in set])
|
385 |
|
|
while True:
|
386 |
|
|
row = bits[-2]
|
387 |
|
|
for bit in row[1:]:
|
388 |
|
|
if bit != row[0]:
|
389 |
|
|
break
|
390 |
|
|
else:
|
391 |
|
|
del bits[-2]
|
392 |
|
|
continue
|
393 |
|
|
break
|
394 |
|
|
set = zip(
|
395 |
|
|
["_".join(name) for name in apply(zip, bits[-2:])],
|
396 |
|
|
[job for bits, job in set])
|
397 |
|
|
for name, job in set:
|
398 |
|
|
warn("building %s as %s" % (job.path, name))
|
399 |
|
|
job.basename = name
|
400 |
|
|
# XXX keep this check until we're properly general
|
401 |
|
|
names = {}
|
402 |
|
|
for job in jobs:
|
403 |
|
|
name = job.basename
|
404 |
|
|
if names.has_key(name):
|
405 |
|
|
raise Error, "%s: duplicate jobname" % name
|
406 |
|
|
names[name] = 1
|
407 |
|
|
|
408 |
|
|
def system(command):
|
409 |
|
|
"""Execute a command."""
|
410 |
|
|
status = os.spawnv(os.P_WAIT, command[0], command)
|
411 |
|
|
if status > 0:
|
412 |
|
|
raise Error, "%s exited with code %d" % (command[0], status)
|
413 |
|
|
elif status < 0:
|
414 |
|
|
raise Error, "%s killed by signal %d" % (command[0], -status)
|
415 |
|
|
|
416 |
|
|
def warn(msg):
|
417 |
|
|
"""Print a warning message."""
|
418 |
|
|
print >>sys.stderr, "%s: warning: %s" % (
|
419 |
|
|
os.path.basename(sys.argv[0]), msg)
|
420 |
|
|
|
421 |
|
|
def classname(bytes):
|
422 |
|
|
"""Extract the class name from the bytes of a class file."""
|
423 |
|
|
klass = classfile.Class(bytes)
|
424 |
|
|
return klass.constants[klass.constants[klass.name][1]][1]
|