Commit 5755a6b3 authored by Kirill Smelkov's avatar Kirill Smelkov

Basic setup.py / Makefile to build/install/sdist stuff + bigfile.so skeleton

It is an early project decision to use gnu99 & Plan9 C extensions, to
simplify C code.

So far we only build stub wendelin/bigfile/_bigfile.so .

Makefile is introduced because there will be targets which are easier to
handle at make level. For end users no make knowledge is required -
usual `python setup.py build|install|...` work, redirecting to make
where necessary.

As was promised (e870781d "Top-level in-tree import redirector")
setup.py contains install-time hooks to handle in-tree wendelin.py and
install it as a module namespace.

For sdist, we just use `git ls-files` info if we are in a checkout.
parent 0b7b5f6a
# Wendelin.core | Instructions to build & test
# Copyright (C) 2014-2015 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
all :
PYTHON ?= python
# use the same C compiler as python
# (for example it could be `gcc -m64` on a 32bit userspace)
CC := $(shell $(PYTHON) -c "from __future__ import print_function; \
import sysconfig as _; \
print(_.get_config_vars()['CC'])")
ifeq ($(CC),)
$(error "Cannot defermine py-CC")
endif
all : bigfile/_bigfile.so
bigfile/_bigfile.so : 3rdparty/ccan/config.h FORCE
$(PYTHON) setup.py ll_build_ext --inplace
FORCE :
# TODO add FORCE?
3rdparty/ccan/config.h: 3rdparty/ccan/Makefile
$(MAKE) -C $(@D) $(@F)
# if there is no ccan/Makefile - ccan submodule has not been initialized
# if there is - it is ok, as that Makefile does not need to be rebuilt
3rdparty/ccan/Makefile:
@echo 'E: 3rdparty/ccan submodule not initialized'
@echo 'E: please do `git submodule update --init`'
@false
/* Wendelin.bigfile | Python interface to memory/files
* Copyright (C) 2014-2015 Nexedi SA and Contributors.
* Kirill Smelkov <kirr@nexedi.com>
*
* This program is free software: you can Use, Study, Modify and Redistribute
* it under the terms of the GNU General Public License version 3, or (at your
* option) any later version, as published by the Free Software Foundation.
*
* You can also Link and Combine this program with other software covered by
* the terms of any of the Open Source Initiative approved licenses and Convey
* the resulting work. Corresponding source of such a combination shall include
* the source code for all other software used.
*
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
*
* See COPYING file for full licensing terms.
*/
#include <Python.h>
static /*const*/ PyMethodDef pybigfile_modulemeths[] = {
{NULL}
};
/* module init */
#if PY_MAJOR_VERSION >= 3
static /*const*/ PyModuleDef pybigfile_moduledef = {
PyModuleDef_HEAD_INIT,
.m_name = "_bigfile",
.m_size = -1, /* = disable creating several instances of this module */
.m_methods = pybigfile_modulemeths,
};
#endif
static PyObject *
_init_bigfile(void)
{
PyObject *m;
#if PY_MAJOR_VERSION >= 3
m = PyModule_Create(&pybigfile_moduledef);
#else
m = Py_InitModule("_bigfile", pybigfile_modulemeths);
#endif
if (!m)
return NULL;
return m;
}
#if PY_MAJOR_VERSION < 3
# define PyInit__bigfile init_bigfile
#endif
__attribute__((visibility("default"))) // XXX should be in PyMODINIT_FUNC
PyMODINIT_FUNC
PyInit__bigfile(void)
{
#if PY_MAJOR_VERSION >= 3
return
#endif
_init_bigfile();
}
# Wendelin.core | pythonic package setup
# Copyright (C) 2014-2015 Nexedi SA and Contributors.
# Kirill Smelkov <kirr@nexedi.com>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Open Source Initiative approved licenses and Convey
# the resulting work. Corresponding source of such a combination shall include
# the source code for all other software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
from setuptools import setup, Extension, Command, find_packages
from setuptools.command.build_py import build_py as _build_py
from setuptools.command.build_ext import build_ext as _build_ext
from distutils.errors import DistutilsExecError
from subprocess import check_output as runcmd
import os
_bigfile = Extension('wendelin.bigfile._bigfile',
sources = [
'bigfile/_bigfile.c',
],
include_dirs = [
'./include',
'./3rdparty/ccan',
'./3rdparty/include'
],
define_macros = [('_GNU_SOURCE',None)],
extra_compile_args = [
'-std=gnu99', # declarations inside for-loop
'-fplan9-extensions', # anonymous-structs + simple inheritance
'-fvisibility=hidden', # by default symbols not visible outside DSO
],
# can't - at runtime links with either python (without libpython) or libpython
# linking with both libpython and python would be wrong
#extra_link_args = [
# '-Wl,--no-undefined', # check DSO for undefined symbols at link time
#]
)
# build_py that
# - prevents in-tree wendelin.py & setup.py to be installed
# - synthesizes wendelin/__init__.py on install
class build_py(_build_py):
def find_package_modules(self, package, package_dir):
modules = _build_py.find_package_modules(self, package, package_dir)
try:
modules.remove(('wendelin', 'wendelin', 'wendelin.py'))
modules.remove(('wendelin', 'setup', 'setup.py'))
except ValueError:
pass # was not there
return modules
def build_packages(self):
_build_py.build_packages(self)
# emit std namespaceing mantra to wendelin/__init__.py
self.initfile = self.get_module_outfile(self.build_lib, ('wendelin',), '__init__')
with open(self.initfile, 'w') as f:
f.write("# this is a namespace package (autogenerated)\n")
f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
def get_outputs(self, include_bytecode=1):
outputs = _build_py.get_outputs(self, include_bytecode)
# add synthethized __init__.py to outputs, so that `pip uninstall`
# works without leaving it
outputs.append(self.initfile)
if include_bytecode:
if self.compile:
outputs.append(self.initfile + 'c')
if self.optimize:
outputs.append(self.initfile + 'o')
return outputs
# run `make <target>`
def runmake(target):
err = os.system('make %s' % target)
if err:
raise DistutilsExecError('Failed to execute `make %s`' % target)
# create distutils command to "run `make <target>`"
def viamake(target, help_text):
class run_viamake(Command):
user_options = []
def initialize_options(self): pass
def finalize_options(self): pass
description = help_text
def run(self):
runmake(target)
return run_viamake
# build_ext that
# - builds via Makefile (and thus pre-builds ccan) XXX hacky
class build_ext(_build_ext):
def run(self):
runmake('all')
return _build_ext.run(self)
# register `func` as entry point `entryname` in `groupname` in distribution `distname` on the fly
def register_as_entrypoint(func, entryname, groupname, distname):
from pkg_resources import working_set, EntryPoint
dist = working_set.by_key[distname]
entrypoint = EntryPoint(entryname, func.__module__, attrs=(func.__name__,),
extras=(), dist=dist)
group = dist.get_entry_map(groupname)
assert entryname not in group
group[entryname] = entrypoint
def git_lsfiles(dirname):
# FIXME dirname is currently ignored
# XXX non-ascii names, etc
for _ in runcmd(['git', 'ls-files']).splitlines():
yield _.decode('utf8') # XXX utf8 hardcoded
# XXX recursive submodules
for _ in runcmd(['git', 'submodule', 'foreach', '--quiet', \
r'git ls-files | sed "s|\(.*\)\$|$path/\1|g"']).splitlines():
yield _.decode('utf8') # XXX utf8 hardcoded
# if we are in git checkout - inject git_lsfiles to setuptools.file_finders
# entry-point on the fly.
#
# otherwise, for released sdist tarball we do not - that tarball already
# contains generated SOURCES.txt and sdist, if run from unpacked tarball, would
# reuse that information.
#
# (to setuptools - beause we have to use some distribution and we already
# depend on setuptools)
if os.path.exists('.git'): # FIXME won't work if we are checked out as e.g. submodule
register_as_entrypoint(git_lsfiles, 'git', 'setuptools.file_finders', 'setuptools')
setup(
name = 'wendelin.core',
description = 'Out-of-core NumPy arrays',
url = 'http://www.wendelin.io/',
license = 'GPLv3+ with wide exception for Open-Source',
author = 'Kirill Smelkov',
author_email= 'kirr@nexedi.com',
keywords = 'bigdata out-of-core numpy virtual-memory',
ext_modules = [_bigfile],
package_dir = {'wendelin': ''},
packages = ['wendelin'] + ['wendelin.%s' % _ for _ in
find_packages(exclude='3rdparty')],
cmdclass = {'build_ext': build_ext,
'll_build_ext': _build_ext, # original build_ext for Makefile
'build_py': build_py,
},
)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment