1 """This module contains the main part of Instant, the build_module function."""
2
3 import os, sys, shutil, glob
4 from itertools import chain
5
6
7 from output import *
8 from config import header_and_libs_from_pkgconfig
9 from paths import *
10 from signatures import *
11 from cache import *
12 from codegeneration import *
13
14
16 instant_assert(isinstance(x, str),
17 "In instant.build_module: Expecting string.")
18
20 instant_assert(isinstance(x, bool),
21 "In instant.build_module: Expecting bool.")
22
24 instant_assert(isinstance(x, (list, tuple)),
25 "In instant.build_module: Expecting sequence.")
26 instant_assert(all(isinstance(i, str) for i in x),
27 "In instant.build_module: Expecting sequence of strings.")
28
32
34 if isinstance(x, str):
35 x = x.split()
36 return strip_strings(x)
37
38
40 """Copy a list of files from a source directory to a destination directory.
41 This may seem a bit complicated, but a lot of this code is error checking."""
42 if os.path.exists(dest):
43 overwriting = set(files) & set(glob.glob(os.path.join(dest, "*")))
44 if overwriting:
45 instant_warning("In instant.copy_files: Path '%s' already exists, "\
46 "overwriting existing files: %r." % (dest, list(overwriting)))
47 else:
48 os.mkdir(dest)
49
50 if source != dest:
51 instant_debug("In instant.copy_files: Copying files %r from %r to %r"\
52 % (files, source, dest))
53
54 for f in files:
55 a = os.path.join(source, f)
56 b = os.path.join(dest, f)
57 instant_assert(a != b, "In instant.copy_files: Seems like the "\
58 "input files are absolute paths, should be relative to "\
59 "source. (%r, %r)" % (a, b))
60 instant_assert(os.path.isfile(a), "In instant.copy_files: "\
61 "Missing source file '%s'." % a)
62 if os.path.isfile(b):
63 os.remove(b)
64 shutil.copyfile(a, b)
65
66
67 -def recompile(modulename, module_path, setup_name, new_compilation_checksum):
68 """Recompile module if the new checksum is different from
69 the one in the checksum file in the module directory."""
70
71 need_recompilation = True
72 compilation_checksum_filename = "%s.checksum" % modulename
73 if os.path.exists(compilation_checksum_filename):
74 checksum_file = open(compilation_checksum_filename)
75 old_compilation_checksum = checksum_file.readline()
76 checksum_file.close()
77 if old_compilation_checksum == new_compilation_checksum:
78 return
79
80
81 (swig_stat, swig_out) = get_status_output("swig -version")
82 if swig_stat != 0:
83 instant_error("In instant.recompile: Could not find swig!"\
84 " You can download swig from http://www.swig.org")
85
86
87 compile_log_filename = os.path.join(module_path, "compile.log")
88 compile_log_file = open(compile_log_filename, "w")
89 try:
90
91 cmd = "python %s build_ext" % setup_name
92 instant_info("--- Instant: compiling ---")
93 instant_debug("cmd = %s" % cmd)
94 ret, output = get_status_output(cmd)
95 compile_log_file.write(output)
96 compile_log_file.flush()
97 if ret != 0:
98 if os.path.exists(compilation_checksum_filename):
99 os.remove(compilation_checksum_filename)
100 instant_error("In instant.recompile: The module did not "\
101 "compile, see '%s'" % compile_log_filename)
102
103
104 cmd = "python %s install --install-platlib=." % setup_name
105 instant_debug("cmd = %s" % cmd)
106 ret, output = get_status_output(cmd)
107 compile_log_file.write(output)
108 compile_log_file.flush()
109 if ret != 0:
110 if os.path.exists(compilation_checksum_filename):
111 os.remove(compilation_checksum_filename)
112 instant_error("In instant.recompile: Could not 'install' "\
113 "the module, see '%s'" % compile_log_filename)
114 finally:
115 compile_log_file.close()
116
117
118 write_file(compilation_checksum_filename, new_compilation_checksum)
119
120
122 "Copy module directory to cache."
123
124 cache_module_path = os.path.join(cache_dir, modulename)
125 if os.path.exists(cache_module_path):
126
127
128 instant_warning("In instant.build_module: Path '%s' already exists,"\
129 " but module wasn't found in cache previously. Overwriting."\
130 % cache_module_path)
131 shutil.rmtree(cache_module_path, ignore_errors=True)
132
133
134 instant_assert(os.path.isdir(module_path), "In instant.build_module:"\
135 " Cannot copy non-existing directory %r!" % module_path)
136 instant_assert(not os.path.isdir(cache_module_path),
137 "In instant.build_module: Cache directory %r shouldn't exist "\
138 "at this point!" % cache_module_path)
139 instant_debug("In instant.build_module: Copying built module from %r"\
140 " to cache at %r" % (module_path, cache_module_path))
141
142
143 shutil.copytree(module_path, cache_module_path)
144 delete_temp_dir()
145 return cache_module_path
146
147
148 -def build_module(modulename=None, source_directory=".",
149 code="", init_code="",
150 additional_definitions="", additional_declarations="",
151 sources=[], wrap_headers=[],
152 local_headers=[], system_headers=[],
153 include_dirs=['.'], library_dirs=[], libraries=[],
154 swigargs=['-c++', '-fcompact', '-O', '-I.', '-small'],
155 cppargs=['-O2'], lddargs=[],
156 object_files=[], arrays=[],
157 generate_interface=True, generate_setup=True,
158 signature=None, cache_dir=None):
159 """Generate and compile a module from C/C++ code using SWIG.
160
161 Arguments:
162 ==========
163 The keyword arguments are as follows:
164 - B{modulename}:
165 - The name you want for the module.
166 If specified, the module will not be cached.
167 If missing, a name will be constructed based on
168 a checksum of the other arguments, and the module
169 will be placed in the global cache. String.
170 - B{source_directory}:
171 - The directory where used supplied files reside.
172 - B{code}:
173 - A string containing C or C++ code to be compiled and wrapped.
174 - B{init_code}:
175 - Code that should be executed when the instant module is imported.
176 - B{additional_definitions}:
177 - A list of additional definitions (typically needed for inheritance).
178 - B{additional_declarations}:
179 - A list of additional declarations (typically needed for inheritance).
180 - B{sources}:
181 - A list of source files to compile and link with the module.
182 - B{wrap_headers}:
183 - A list of local header files that should be wrapped by SWIG.
184 - B{local_headers}:
185 - A list of local header files required to compile the wrapped code.
186 - B{system_headers}:
187 - A list of system header files required to compile the wrapped code.
188 - B{include_dirs}:
189 - A list of directories to search for header files.
190 - B{library_dirs}:
191 - A list of directories to search for libraries (C{-l}).
192 - B{libraries}:
193 - A list of libraries needed by the instant module.
194 - B{swigargs}:
195 - List of arguments to swig, e.g. C{["-lpointers.i"]}
196 to include the SWIG pointers.i library.
197 - B{cppargs}:
198 - List of arguments to the compiler, e.g. C{["-D", "-U"]}.
199 - B{lddargs}:
200 - List of arguments to the linker, e.g. C{["-D", "-U"]}.
201 - B{object_files}:
202 - If you want to compile the files yourself. TODO: Not yet supported.
203 - B{arrays}:
204 - A list of the C arrays to be made from NumPy arrays.
205 FIXME: Describe this correctly. Tests pass arrays of arrays of strings.
206 - B{generate_interface}:
207 - A bool to indicate if you want to generate the interface files.
208 - B{generate_setup}:
209 - A bool to indicate if you want to generate the setup.py file.
210 - B{signature}:
211 - A signature string to identify the form instead of the source code.
212 - B{cache_dir}:
213 - A directory to look for cached modules and place new ones.
214 If missing, a default directory is used. Note that the module
215 will not be cached if C{modulename} is specified.
216 The cache directory should not be used for anything else.
217 """
218
219
220 original_path = os.getcwd()
221
222
223
224 instant_assert(modulename is None or isinstance(modulename, str),
225 "In instant.build_module: Expecting modulename to be string or None.")
226 assert_is_str(source_directory)
227 source_directory = os.path.abspath(source_directory)
228 assert_is_str(code)
229 assert_is_str(init_code)
230 assert_is_str(additional_definitions)
231 assert_is_str(additional_declarations)
232 sources = strip_strings(sources)
233 wrap_headers = strip_strings(wrap_headers)
234 local_headers = strip_strings(local_headers)
235 system_headers = strip_strings(system_headers)
236 include_dirs = strip_strings(include_dirs)
237 library_dirs = strip_strings(library_dirs)
238 libraries = strip_strings(libraries)
239 swigargs = arg_strings(swigargs)
240 cppargs = arg_strings(cppargs)
241 lddargs = arg_strings(lddargs)
242 object_files = strip_strings(object_files)
243 arrays = [strip_strings(a) for a in arrays]
244 assert_is_bool(generate_interface)
245 assert_is_bool(generate_setup)
246 instant_assert( signature is None \
247 or isinstance(signature, str) \
248 or hasattr(signature, "signature"),
249 "In instant.build_module: Expecting modulename to be string or None.")
250 instant_assert(not (signature is not None and modulename is not None),
251 "In instant.build_module: Can't have both modulename and signature.")
252
253
254
255 cache_dir = validate_cache_dir(cache_dir)
256
257
258 csrcs = [f for f in sources if f.endswith('.c') or f.endswith('.C')]
259 cppsrcs = [f for f in sources if f.endswith('.cpp') or f.endswith('.cxx')]
260 if csrcs:
261 instant_error("FIXME: setup.py doesn't use the C sources.")
262 instant_assert(len(csrcs) + len(cppsrcs) == len(sources),
263 "In instant.build_module: Source files must have '.c' or '.cpp' suffix")
264
265
266 instant_debug('In instant.build_module:')
267 instant_debug('::: Begin Arguments :::')
268 instant_debug(' modulename: %r' % modulename)
269 instant_debug(' code: %r' % code)
270 instant_debug(' init_code: %r' % init_code)
271 instant_debug(' additional_definitions: %r' % additional_definitions)
272 instant_debug(' additional_declarations: %r' % additional_declarations)
273 instant_debug(' sources: %r' % sources)
274 instant_debug(' csrcs: %r' % csrcs)
275 instant_debug(' cppsrcs: %r' % cppsrcs)
276 instant_debug(' wrap_headers: %r' % wrap_headers)
277 instant_debug(' local_headers: %r' % local_headers)
278 instant_debug(' system_headers: %r' % system_headers)
279 instant_debug(' include_dirs: %r' % include_dirs)
280 instant_debug(' library_dirs: %r' % library_dirs)
281 instant_debug(' libraries: %r' % libraries)
282 instant_debug(' swigargs: %r' % swigargs)
283 instant_debug(' cppargs: %r' % cppargs)
284 instant_debug(' lddargs: %r' % lddargs)
285 instant_debug(' object_files: %r' % object_files)
286 instant_debug(' arrays: %r' % arrays)
287 instant_debug(' generate_interface: %r' % generate_interface)
288 instant_debug(' generate_setup: %r' % generate_setup)
289 instant_debug(' signature: %r' % signature)
290 instant_debug(' cache_dir: %r' % cache_dir)
291 instant_debug('::: End Arguments :::')
292
293
294
295
296
297 if modulename is None:
298
299 if signature is None:
300
301
302
303 checksum_args = ( \
304
305
306
307
308 code, init_code,
309 additional_definitions,
310 additional_declarations,
311
312
313
314 system_headers,
315 include_dirs, library_dirs, libraries,
316 swigargs, cppargs, lddargs,
317 object_files, arrays,
318 generate_interface, generate_setup,
319
320
321 )
322 allfiles = sources + wrap_headers + local_headers
323 text = "\n".join((str(a) for a in checksum_args))
324 signature = modulename_from_checksum(compute_checksum(text, allfiles))
325 modulename = signature
326 moduleids = [signature]
327 else:
328 module, moduleids = check_memory_cache(signature)
329 if module: return module
330 modulename = moduleids[-1]
331
332
333 module = check_disk_cache(modulename, cache_dir, moduleids)
334 if module: return module
335
336
337 module_path = os.path.join(get_temp_dir(), modulename)
338 instant_assert(not os.path.exists(module_path),
339 "In instant.build_module: Not expecting module_path to exist: '%s'"\
340 % module_path)
341 os.mkdir(module_path)
342 use_cache = True
343 else:
344 use_cache = False
345 moduleids = []
346 module_path = os.path.join(original_path, modulename)
347 if not os.path.exists(module_path):
348 os.mkdir(module_path)
349
350
351
352
353
354
355
356
357
358
359
360 try:
361
362
363 module_path = os.path.abspath(module_path)
364 files_to_copy = sources + wrap_headers + local_headers + object_files
365 copy_files(source_directory, module_path, files_to_copy)
366
367
368
369 os.chdir(module_path)
370
371
372 write_file("__init__.py", "from %s import *" % modulename)
373
374
375 ifile_name = "%s.i" % modulename
376 if generate_interface:
377 write_interfacefile(ifile_name, modulename, code, init_code,
378 additional_definitions, additional_declarations, system_headers,
379 local_headers, wrap_headers, arrays)
380
381
382 setup_name = "setup.py"
383 if generate_setup:
384 write_setup(setup_name, modulename, csrcs, cppsrcs, local_headers,
385 include_dirs, library_dirs, libraries, swigargs, cppargs, lddargs)
386
387
388
389
390
391
392
393
394
395
396
397
398 checksum_args = ( \
399
400
401
402
403
404
405
406
407
408
409 system_headers,
410 include_dirs, library_dirs, libraries,
411 swigargs, cppargs, lddargs,
412 object_files,
413
414
415
416
417 )
418 text = "\n".join((str(a) for a in checksum_args))
419 allfiles = sources + wrap_headers + local_headers + [ifile_name]
420 new_compilation_checksum = compute_checksum(text, allfiles)
421
422
423 recompile(modulename, module_path, setup_name, new_compilation_checksum)
424
425
426
427
428 if use_cache:
429 module_path = copy_to_cache(module_path, cache_dir, modulename)
430
431
432 module = import_and_cache_module(module_path, modulename, moduleids)
433 if not module:
434 instant_error("Failed to import newly compiled module!")
435
436 instant_debug("In instant.build_module: Returning %s from build_module."\
437 % module)
438 return module
439
440
441 finally:
442
443 os.chdir(original_path)
444
445 instant_error("In instant.build_module: Should never reach this point!")
446
447