First part of reorg source tree for expandability.

Move 3rd party libraries to deps/ directory.

Move gevent and greentest to src/ directory. This fixes tox.

At this point the builds should all work the same. Lets check.

.gitignore

@@ -5,18 +5,18 @@ build/
 *.so
 *.egg-info
 gevent.*.[ch]
-gevent/corecext.pyx
-gevent/__pycache__
-gevent/libev
-gevent/_corecffi.c
-gevent/_corecffi.o
+src/gevent/corecext.pyx
+src/gevent/__pycache__
+src/gevent/libev
+src/gevent/_corecffi.c
+src/gevent/_corecffi.o
 Makefile.ext
 MANIFEST
 *_flymake.py
 
-greentest/.coverage\.*
-greentest/htmlcov
-greentest/.coverage
+src/greentest/.coverage\.*
+src/greentest/htmlcov
+src/greentest/.coverage
 
 doc/changelog.rst
 doc/_build
@@ -33,40 +33,40 @@ doc/gevent.*.rst
 !doc/gevent.wsgi.rst
 
 # Artifacts of configuring in place
-c-ares/config.log
-c-ares/config.status
-c-ares/stamp-h1
-c-ares/stamp-h2
-c-ares/ares_build.h.orig
-c-ares/ares_config.h
-c-ares/.libs
-c-ares/*.o
-c-ares/*.lo
-c-ares/*.la
-c-ares/.deps
-c-ares/acountry
-c-ares/adig
-c-ares/ahost
-c-ares/Makefile
-c-ares/libtool
-c-ares/libcares.pc
-c-ares/test/.deps
-c-ares/test/Makefile
-c-ares/test/config.log
-c-ares/test/config.status
-c-ares/test/libtool
-c-ares/test/stamp-h1
-libev/.deps
-libev/Makefile
-libev/config.log
-libev/config.h
-libev/config.status
-libev/libtool
-libev/stamp-h1
-libev/.libs
-libev/*.lo
-libev/*.la
-libev/*.o
+deps/c-ares/config.log
+deps/c-ares/config.status
+deps/c-ares/stamp-h1
+deps/c-ares/stamp-h2
+deps/c-ares/ares_build.h.orig
+deps/c-ares/ares_config.h
+deps/c-ares/.libs
+deps/c-ares/*.o
+deps/c-ares/*.lo
+deps/c-ares/*.la
+deps/c-ares/.deps
+deps/c-ares/acountry
+deps/c-ares/adig
+deps/c-ares/ahost
+deps/c-ares/Makefile
+deps/c-ares/libtool
+deps/c-ares/libcares.pc
+deps/c-ares/test/.deps
+deps/c-ares/test/Makefile
+deps/c-ares/test/config.log
+deps/c-ares/test/config.status
+deps/c-ares/test/libtool
+deps/c-ares/test/stamp-h1
+deps/libev/.deps
+deps/libev/Makefile
+deps/libev/config.log
+deps/libev/config.h
+deps/libev/config.status
+deps/libev/libtool
+deps/libev/stamp-h1
+deps/libev/.libs
+deps/libev/*.lo
+deps/libev/*.la
+deps/libev/*.o
 
 # running setup.py on PyPy
 config.h

MANIFEST.in

-recursive-include greentest *
+recursive-include src/greentest *
 recursive-include examples *
-recursive-include gevent *
+recursive-include src/gevent *
 recursive-include doc *
-recursive-include libev *
-recursive-include c-ares *
+recursive-include deps *
 recursive-include util *
 
 include LICENSE
@@ -40,14 +39,14 @@ global-exclude *.la
 global-exclude config.log config.status
 prune doc/_build
 global-exclude *.pyc
-recursive-exclude greentest .coverage
-prune greentest/htmlcov
-recursive-exclude c-ares stamp-h? ares_build.h.orig
-prune libev/.deps
-recursive-exclude libev Makefile libtool stamp-h? config.h
+recursive-exclude src/greentest .coverage
+prune src/greentest/htmlcov
+recursive-exclude deps/c-ares stamp-h? ares_build.h.orig
+prune deps/libev/.deps
+recursive-exclude deps/libev Makefile libtool stamp-h? config.h
 # This is the output of _corecffi_build.py and may be particular
 # to each CFFI version/platform
-recursive-exclude gevent _corecffi.c
+recursive-exclude src/gevent _corecffi.c
 # See comments in Makefile; this is renamed to Makefile.ext
 # this exclude keeps check-manifest from complaining
 exclude Makefile

Makefile

@@ -12,48 +12,48 @@ export PATH:=$(BUILD_RUNTIMES)/snakepit:$(TOOLS):$(PATH)
 export LC_ALL=C.UTF-8
 
 
-all: gevent/gevent.corecext.c gevent/gevent.ares.c gevent/gevent._semaphore.c
+all: src/gevent/gevent.corecext.c src/gevent/gevent.ares.c src/gevent/gevent._semaphore.c
 
-gevent/gevent.corecext.c: gevent/corecext.ppyx gevent/libev.pxd util/cythonpp.py
-	$(PYTHON) util/cythonpp.py -o gevent.corecext.c gevent/corecext.ppyx
+src/gevent/gevent.corecext.c: src/gevent/corecext.ppyx src/gevent/libev.pxd util/cythonpp.py
+	$(PYTHON) util/cythonpp.py -o gevent.corecext.c src/gevent/corecext.ppyx
 	echo '#include "callbacks.c"' >> gevent.corecext.c
-	mv gevent.corecext.* gevent/
+	mv gevent.corecext.* src/gevent/
 
-gevent/gevent.ares.c: gevent/ares.pyx gevent/*.pxd
+src/gevent/gevent.ares.c: src/gevent/ares.pyx src/gevent/*.pxd
 	$(CYTHON) -o gevent.ares.c gevent/ares.pyx
-	mv gevent.ares.* gevent/
+	mv gevent.ares.* src/gevent/
 
-gevent/gevent._semaphore.c: gevent/_semaphore.py gevent/_semaphore.pxd
+src/gevent/gevent._semaphore.c: src/gevent/_semaphore.py src/gevent/_semaphore.pxd
 # On PyPy, if we wanted to use Cython to compile _semaphore.py, we'd
 # need to have _semaphore named as a .pyx file so it doesn't get
 # loaded in preference to the .so. (We want to keep the definitions
 # separate in a .pxd file for ease of reading, and that only works
 # with .py files, so we'd have to copy them back and forth.)
-#	cp gevent/_semaphore.pyx gevent/_semaphore.py
-	$(CYTHON) -o gevent._semaphore.c gevent/_semaphore.py
-	mv gevent._semaphore.* gevent/
-#	rm gevent/_semaphore.py
+#	cp src/gevent/_semaphore.pyx src/gevent/_semaphore.py
+	$(CYTHON) -o gevent._semaphore.c src/gevent/_semaphore.py
+	mv gevent._semaphore.* src/gevent/
+#	rm src/gevent/_semaphore.py
 
 clean:
-	rm -f corecext.pyx gevent/corecext.pyx
-	rm -f gevent.corecext.c gevent.corecext.h gevent/gevent.corecext.c gevent/gevent.corecext.h
-	rm -f gevent.ares.c gevent.ares.h gevent/gevent.ares.c gevent/gevent.ares.h
-	rm -f gevent._semaphore.c gevent._semaphore.h gevent/gevent._semaphore.c gevent/gevent._semaphore.h
-	rm -f gevent/*.so
-	rm -rf gevent/__pycache__
-	rm -rf gevent/*.pyc
+	rm -f corecext.pyx src/gevent/corecext.pyx
+	rm -f gevent.corecext.c gevent.corecext.h src/gevent/gevent.corecext.c src/gevent/gevent.corecext.h
+	rm -f gevent.ares.c gevent.ares.h src/gevent/gevent.ares.c src/gevent/gevent.ares.h
+	rm -f gevent._semaphore.c gevent._semaphore.h src/gevent/gevent._semaphore.c src/gevent/gevent._semaphore.h
+	rm -f src/gevent/*.so
+	rm -rf src/gevent/__pycache__
+	rm -rf src/gevent/*.pyc
 
 doc:
 	cd doc && PYTHONPATH=.. make html
 
 whitespace:
-	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./greentest/htmlcov/*" -not -path "./greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*" -not -path "./libev/*" -not -path "./gevent/libev/*" -not -path "./gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./c-ares/*" -not -path "./gevent/gevent.*.[ch]" -not -path "./gevent/corecext.pyx" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
+	! find . -not -path "*.pem" -not -path "./.eggs/*" -not -path "./src/greentest/htmlcov/*" -not -path "./src/greentest/.coverage.*" -not -path "./.tox/*" -not -path "*/__pycache__/*" -not -path "*.so" -not -path "*.pyc" -not -path "./.git/*" -not -path "./build/*"  -not -path "./src/gevent/libev/*" -not -path "./gevent.egg-info/*" -not -path "./dist/*" -not -path "./.DS_Store" -not -path "./deps/*" -not -path "./src/gevent/gevent.*.[ch]" -not -path "./src/gevent/corecext.pyx" -not -path "./doc/_build/*" -not -path "./doc/mytheme/static/*" -type f | xargs egrep -l " $$"
 
 prospector:
 	which prospector
 	which pylint
 # debugging
-#	pylint --rcfile=.pylintrc --init-hook="import sys, code; sys.excepthook = lambda exc, exc_type, tb: print(tb.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_frame.f_locals['self'])" gevent greentest/* || true
+#	pylint --rcfile=.pylintrc --init-hook="import sys, code; sys.excepthook = lambda exc, exc_type, tb: print(tb.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_next.tb_frame.f_locals['self'])" gevent src/greentest/* || true
 	${PYTHON} scripts/gprospector.py -X
 
 lint: whitespace prospector
@@ -66,18 +66,18 @@ test_prelim:
 	make bench
 
 toxtest: test_prelim
-	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
+	cd src/greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config known_failures.py
 
 fulltoxtest: test_prelim
-	cd greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config ../known_failures.py
-	cd greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config ../known_failures.py --ignore tests_that_dont_use_resolver.txt
-	cd greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config ../known_failures.py `grep -l subprocess test_*.py`
+	cd src/greentest && GEVENT_RESOLVER=thread ${PYTHON} testrunner.py --config known_failures.py
+	cd src/greentest && GEVENT_RESOLVER=ares GEVENTARES_SERVERS=8.8.8.8 ${PYTHON} testrunner.py --config known_failures.py --ignore tests_that_dont_use_resolver.txt
+	cd src/greentest && GEVENT_FILE=thread ${PYTHON} testrunner.py --config known_failures.py `grep -l subprocess test_*.py`
 
 leaktest:
 	GEVENTSETUP_EV_VERIFY=3 GEVENTTEST_LEAKCHECK=1 make fulltoxtest
 
 bench:
-	${PYTHON} greentest/bench_sendall.py
+	${PYTHON} src/greentest/bench_sendall.py
 
 
 travis_test_linters:
@@ -85,9 +85,9 @@ travis_test_linters:
 	GEVENTTEST_COVERAGE=1 make leaktest
 # because we set parallel=true, each run produces new and different coverage files; they all need
 # to be combined
-	coverage combine . greentest/
+	coverage combine . src/greentest/
 
-	coveralls --rcfile=greentest/.coveragerc
+	coveralls --rcfile=src/greentest/.coveragerc
 
 
 .PHONY: clean all doc prospector whitespace lint travistest travis
@@ -171,6 +171,6 @@ test-py27-cffi: $(PY27)
 	GEVENT_CORE_CFFI_ONLY=1 PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop toxtest
 
 test-py27-noembed: $(PY27)
-	cd libev && ./configure --disable-dependency-tracking && make
-	cd c-ares && ./configure --disable-dependency-tracking && make
-	CPPFLAGS="-Ilibev -Ic-ares" LDFLAGS="-Llibev/.libs -Lc-ares/.libs" LD_LIBRARY_PATH="$(PWD)/libev/.libs:$(PWD)/c-ares/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop toxtest
+	cd deps/libev && ./configure --disable-dependency-tracking && make
+	cd deps/c-ares && ./configure --disable-dependency-tracking && make
+	CPPFLAGS="-Ideps/libev -Ideps/c-ares" LDFLAGS="-Ldeps/libev/.libs -Ldeps/c-ares/.libs" LD_LIBRARY_PATH="$(PWD)/deps/libev/.libs:$(PWD)/deps/c-ares/.libs" EMBED=0 GEVENT_CORE_CEXT_ONLY=1 PYTHON=python2.7 PATH=$(BUILD_RUNTIMES)/versions/python2.7/bin:$(PATH) make develop toxtest

README.rst

@@ -77,8 +77,8 @@ There are a few different ways to run the tests. To simply run the
 tests on one version of Python during development, try this::
 
   python setup.py develop
-  cd greentest
-  PYTHONPATH=.. python testrunner.py --config ../known_failures.py
+  cd src/greentest
+  PYTHONPATH=.. python testrunner.py --config known_failures.py
 
 Before submitting a pull request, it's a good idea to run the tests
 across all supported versions of Python, and to check the code quality
@@ -93,8 +93,8 @@ The testrunner accepts a ``--coverage`` argument to enable code
 coverage metrics through the `coverage.py`_ package. That would go
 something like this::
 
-  cd greentest
-  PYTHONPATH=.. python testrunner.py --config ../known_failures.py --coverage
+  cd src/greentest
+  PYTHONPATH=.. python testrunner.py --config known_failures.py --coverage
   coverage combine
   coverage html -i
   <open htmlcov/index.html>

appveyor.yml

@@ -145,7 +145,7 @@ build_script:
 
 test_script:
   # Run the project tests
-  - "cd greentest && %PYEXE% testrunner.py --config ../known_failures.py && cd .."
+  - "cd src/greentest && %PYEXE% testrunner.py --config known_failures.py && cd ../.."
 
 after_test:
   # We already built the wheel during build_script, because it's

setup.py

@@ -56,7 +56,7 @@ from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatfo
 ext_errors = (CCompilerError, DistutilsExecError, DistutilsPlatformError, IOError)
 
 
-with open('gevent/__init__.py') as _:
+with open('src/gevent/__init__.py') as _:
     __version__ = re.search(r"__version__\s*=\s*'(.*)'", _.read(), re.M).group(1)
 assert __version__
 
@@ -86,8 +86,8 @@ def get_config_value(key, defkey, path):
     return os.path.exists(path)
 
 
-LIBEV_EMBED = get_config_value('LIBEV_EMBED', 'EMBED', 'libev')
-CARES_EMBED = get_config_value('CARES_EMBED', 'EMBED', 'c-ares')
+LIBEV_EMBED = get_config_value('LIBEV_EMBED', 'EMBED', 'deps/libev')
+CARES_EMBED = get_config_value('CARES_EMBED', 'EMBED', 'deps/c-ares')
 
 define_macros = []
 libraries = []
@@ -97,7 +97,7 @@ libraries = []
 # source checkout on pypy, OLDPWD will be the location of setup.py
 # and the PyPy branch will clean it up.
 libev_configure_command = ' '.join([
-    "(cd ", _quoted_abspath('libev/'),
+    "(cd ", _quoted_abspath('deps/libev/'),
     " && /bin/sh ./configure ",
     " && cp config.h \"$OLDPWD\"",
     ")",
@@ -112,7 +112,7 @@ else:
     _m32 = ''
 
 # Use -r, not -e, for support of old solaris. See https://github.com/gevent/gevent/issues/777
-ares_configure_command = ' '.join(["(cd ", _quoted_abspath('c-ares/'),
+ares_configure_command = ' '.join(["(cd ", _quoted_abspath('deps/c-ares/'),
                                    " && if [ -r ares_build.h ]; then cp ares_build.h ares_build.h.orig; fi ",
                                    " && /bin/sh ./configure --disable-dependency-tracking " + _m32 + "CONFIG_COMMANDS= ",
                                    " && cp ares_config.h ares_build.h \"$OLDPWD\" ",
@@ -134,19 +134,19 @@ def expand(*lst):
 
 
 CORE = Extension(name='gevent.corecext',
-                 sources=['gevent/gevent.corecext.c'],
-                 include_dirs=['libev'] if LIBEV_EMBED else [],
+                 sources=['src/gevent/gevent.corecext.c'],
+                 include_dirs=['deps/libev'] if LIBEV_EMBED else [],
                  libraries=libraries,
                  define_macros=define_macros,
-                 depends=expand('gevent/callbacks.*', 'gevent/stathelper.c', 'gevent/libev*.h', 'libev/*.*'))
+                 depends=expand('src/gevent/callbacks.*', 'src/gevent/stathelper.c', 'src/gevent/libev*.h', 'deps/libev/*.*'))
 # QQQ libev can also use -lm, however it seems to be added implicitly
 
 ARES = Extension(name='gevent.ares',
-                 sources=['gevent/gevent.ares.c'],
-                 include_dirs=['c-ares'] if CARES_EMBED else [],
+                 sources=['src/gevent/gevent.ares.c'],
+                 include_dirs=['deps/c-ares'] if CARES_EMBED else [],
                  libraries=libraries,
                  define_macros=define_macros,
-                 depends=expand('gevent/dnshelper.c', 'gevent/cares_*.*'))
+                 depends=expand('src/gevent/dnshelper.c', 'src/gevent/cares_*.*'))
 ARES.optional = True
 
 
@@ -210,7 +210,7 @@ def configure_ares(bext, ext):
         os.makedirs(bdir)
 
     if WIN:
-        shutil.copy("c-ares\\ares_build.h.dist", os.path.join(bdir, "ares_build.h"))
+        shutil.copy("deps\\c-ares\\ares_build.h.dist", os.path.join(bdir, "ares_build.h"))
         return
 
     cwd = os.getcwd()
@@ -248,12 +248,12 @@ else:
 
 
 if CARES_EMBED:
-    ARES.sources += expand('c-ares/*.c')
+    ARES.sources += expand('deps/c-ares/*.c')
     # Strip the standalone binaries that would otherwise
     # cause linking issues
     for bin_c in ('acountry', 'adig', 'ahost'):
         try:
-            ARES.sources.remove('c-ares/' + bin_c + '.c')
+            ARES.sources.remove('deps/c-ares/' + bin_c + '.c')
         except ValueError:
             pass
     ARES.configure = configure_ares
@@ -324,25 +324,25 @@ class my_build_ext(build_ext):
                 raise BuildFailed
             else:
                 raise
-        if not PYPY:
-            self.gevent_symlink(ext)
+        # if not PYPY:
+        #     self.gevent_symlink(ext)
         return result
 
-    def gevent_symlink(self, ext):
-        # hack: create a symlink from build/../core.so to gevent/core.so
-        # to prevent "ImportError: cannot import name core" failures
-        try:
-            fullname = self.get_ext_fullname(ext.name)
-            modpath = fullname.split('.')
-            filename = self.get_ext_filename(ext.name)
-            filename = os.path.split(filename)[-1]
-            if not self.inplace:
-                filename = os.path.join(*modpath[:-1] + [filename])
-                path_to_build_core_so = os.path.join(self.build_lib, filename)
-                path_to_core_so = join('gevent', basename(path_to_build_core_so))
-                link(path_to_build_core_so, path_to_core_so)
-        except Exception:
-            traceback.print_exc()
+    # def gevent_symlink(self, ext):
+    #     # hack: create a symlink from build/../core.so to gevent/core.so
+    #     # to prevent "ImportError: cannot import name core" failures
+    #     try:
+    #         fullname = self.get_ext_fullname(ext.name)
+    #         modpath = fullname.split('.')
+    #         filename = self.get_ext_filename(ext.name)
+    #         filename = os.path.split(filename)[-1]
+    #         if not self.inplace:
+    #             filename = os.path.join(*modpath[:-1] + [filename])
+    #             path_to_build_core_so = os.path.join(self.build_lib, filename)
+    #             path_to_core_so = join('gevent', basename(path_to_build_core_so))
+    #             link(path_to_build_core_so, path_to_core_so)
+    #     except Exception:
+    #         traceback.print_exc()
 
 
 def link(source, dest):
@@ -373,7 +373,7 @@ def read(name, *args):
     except OSError:
         return ''
 
-cffi_modules = ['gevent/_corecffi_build.py:ffi']
+cffi_modules = ['src/gevent/_corecffi_build.py:ffi']
 
 if PYPY:
     install_requires = []
@@ -425,7 +425,8 @@ elif PYPY:
         # won't do it (since we're not building it)
         system(libev_configure_command)
         # Then get rid of the extra copy created in place
-        system('rm config.h')
+        # XXX no more
+        #system('rm config.h')
     # NOTE that we're NOT adding the distutils extension module, as
     # doing so compiles the module already: import gevent._corecffi_build
     # imports gevent, which imports the hub, which imports the core,
@@ -450,13 +451,13 @@ elif PYPY:
         #          sources=["gevent/gevent._semaphore.c"]),
     ]
     include_package_data = True
-    run_make = 'gevent/gevent.ares.c'
+    run_make = 'src/gevent/gevent.ares.c'
 else:
     ext_modules = [
         CORE,
         ARES,
         Extension(name="gevent._semaphore",
-                  sources=["gevent/gevent._semaphore.c"]),
+                  sources=["src/gevent/gevent._semaphore.c"]),
     ]
     include_package_data = False
     run_make = True
@@ -491,6 +492,7 @@ def run_setup(ext_modules, run_make):
         maintainer='Jason Madden',
         maintainer_email='jason@nextthought.com',
         url='http://www.gevent.org/',
+        package_dir={'': 'src'},
         packages=['gevent'],
         include_package_data=include_package_data,
         ext_modules=ext_modules,

gevent/_corecffi_build.py → src/gevent/_corecffi_build.py

@@ -61,7 +61,7 @@ void vfd_free(int);
 
 include_dirs = [
     thisdir, # libev_vfd.h
-    os.path.abspath(os.path.join(thisdir, '..', 'libev')),
+    os.path.abspath(os.path.join(thisdir, '..', '..', 'deps', 'libev')),
 ]
 ffi.cdef(_cdef)
 ffi.set_source('gevent._corecffi', _source, include_dirs=include_dirs)

tox.ini

@@ -13,16 +13,6 @@ whitelist_externals =
     *
 commands =
     make toxtest
-# our Makefile assumes it's being run from the directory
-# containing setup.py. However, that directory also has the "gevent"
-# directory in it. Which means it's on sys.path first. So unless it
-# has the correct binary extensions for the python version/impl we're trying
-# to run, nothing works. By setting usedevelop=True, then the .so gets built
-# in place in that gevent directory, which makes the tox commands run. This
-# can step on the toes of various implementations when they share the same name
-# for files, but at least it lets some implementations co-exist.
-# The real solution is probably to put "gevent" beneath a "src" directory.
-usedevelop = True
 
 [testenv:py33]
 # On OS X, at least, the binary wheel for 1.5.2 is broken