new software: ProFTPd

Proftpd with sftp and virtual users

http://www.proftpd.org/docs/

# Features

 * sftp only is enabled
 * partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
 * 5 failed login attempts will cause the host to be temporary banned


# TODO

 * only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
 * log rotation
 * make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
 * make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
 * allow configuring webhooks when new file is uploaded



----

depends on nexedi/slapos.core!41

/reviewed-on nexedi/slapos!311

component/proftpd/buildout.cfg

+# http://www.proftpd.org/ - Highly configurable GPL-licensed FTP server software
+#
+# Because it uses collective.recipe.environment this components needs slapos.core >= 1.4.7
+# ( grid: do not hide `$USER` when running buildout )
+#
+[buildout]
+extends = 
+  ../openssl/buildout.cfg
+  ../perl/buildout.cfg
+  ../zlib/buildout.cfg
+  ../curl/buildout.cfg
+  ../libtool/buildout.cfg
+  ../git/buildout.cfg
+
+# proftpd server
+[proftpd-environment]
+recipe = collective.recipe.environment
+
+[proftpd-grp]
+recipe = collective.recipe.grp
+
+[proftpd]
+recipe = slapos.recipe.cmmi
+md5sum = 13270911c42aac842435f18205546a1b
+url = ftp://ftp.proftpd.org/distrib/source/proftpd-1.3.6.tar.gz
+configure-options =
+  --enable-openssl
+  --enable-nls
+  --enable-ctrls
+  --enable-dso
+  --with-modules=mod_sftp:mod_ban
+  --prefix=${buildout:parts-directory}/${:_buildout_section_name_}
+environment = 
+  CPPFLAGS=${:cppflags}
+  LDFLAGS=${:ldflags}
+  install_user=${proftpd-environment:USER}
+  install_group=${proftpd-grp:GROUP}
+cppflags=-I${zlib:location}/include -I${openssl:location}/include
+ldflags=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib
+
+
+# mod_auth_web: a proftpd module to authenticate users against an HTTP service
+[proftpd-mod_auth_web-repository]
+recipe = slapos.recipe.build:gitclone
+#repository = https://github.com/proftpd/mod_auth_web
+# XXX until https://github.com/proftpd/mod_auth_web/pull/1 gets merged, we use
+# the source repository of this PR directly
+repository = https://github.com/jwm/mod_auth_web
+revision = dec090bd0e287544a34be156ee17f715bd4286f9
+git-executable = ${git:location}/bin/git
+
+[proftpd-mod_auth_web]
+recipe = plone.recipe.command
+stop-on-error = true
+command =
+  LIBTOOL=${libtool:location}/bin/libtool \
+  LDFLAGS="-Wl,-rpath=${curl:location}/lib ${proftpd:ldflags}" \
+  ${proftpd-output:prxs} -c -i \
+  -I ${curl:location}/include ${proftpd:cppflags} \
+  -L ${curl:location}/lib/ \
+  -l curl \
+  -d ${proftpd-mod_auth_web-repository:location}/mod_auth_web.c
+location=${proftpd:location}/libexec/mod_auth_web.so
+
+
+# ftpasswod: a perl script to manage a proftpd AuthUserFile
+[ftpasswd]
+recipe = slapos.recipe.build:download
+url = https://raw.githubusercontent.com/proftpd/proftpd/v1.3.6/contrib/ftpasswd
+md5sum = 4a47df2cab86d8de7077a445bb416f31
+download-only = true
+mode = 0700
+
+
+[proftpd-output]
+# Shared binary location to ease migration
+recipe = plone.recipe.command
+stop-on-error = true
+update-command = ${:command}
+command = ${coreutils-output:test} -x ${:proftpd} -a -x ${:ftpasswd}
+modules-deps =
+  ${proftpd-mod_auth_web:recipe}
+
+perl = ${perl:location}/bin/perl
+ftpasswd = ${ftpasswd:target}
+proftpd = ${proftpd:location}/sbin/proftpd
+prxs = ${proftpd:location}/bin/prxs
+ftpdctl = ${proftpd:location}/bin/ftpdctl

software/erp5testnode/testsuite/proftpd/README.md

+# ProFTPd test
+
+This software release is simply to run the test suite from `../proftpd/test/setup.py`
+
+Nexedi staff can see the results of this test from the test suite
+`SLAPOS-PROFTPD-TEST` in test result module.
+

software/erp5testnode/testsuite/proftpd/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template]
+filename = instance.cfg
+md5sum = 9b4cc727c43d0daaec5b01cf4b9f7a15
+

software/erp5testnode/testsuite/proftpd/instance.cfg

+[buildout]
+parts =
+  slapos-test-runner
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[download-source]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+
+[slapos]
+<= download-source
+repository = ${slapos-repository:location}
+
+[create-directory]
+recipe = slapos.cookbook:mkdirectory
+bin = $${buildout:directory}/bin
+
+[slapos-test-runner]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${create-directory:bin}/runTestSuite
+command-line =
+  ${buildout:bin-directory}/runTestSuite
+  --python_interpreter=${buildout:bin-directory}/${eggs:interpreter}
+  --source_code_path_list=$${slapos:location}/software/proftpd/test
+
+
+# XXX slapos.cookbook:wrapper does not allow extending env, so we add some default $PATH entries ( not sure they are needed )
+environment =
+  PATH=${buildout:bin-directory}:/usr/bin/:/bin/
+  LOCAL_IPV4=$${slap-configuration:ipv4-random}
+  GLOBAL_IPV6=$${slap-configuration:ipv6-random}

software/erp5testnode/testsuite/proftpd/software.cfg

+[buildout]
+
+extends =
+  ../../../../component/git/buildout.cfg
+  ../../../../stack/slapos.cfg
+  ./buildout.hash.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  template
+
+[setup-develop-egg]
+recipe = zc.recipe.egg:develop
+
+[slapos.test.proftpd-setup]
+<= setup-develop-egg
+egg = slapos.test.proftpd
+setup = ${slapos-repository:location}/software/proftpd/test/
+
+[erp5.util-setup]
+<= setup-develop-egg
+egg = erp5.util[testnode]
+setup = ${erp5.util-repository:location}
+
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  ${slapos.test.proftpd-setup:egg}
+  ${erp5.util-setup:egg}
+  slapos.core
+entry-points =
+  runTestSuite=erp5.util.testsuite:runTestSuite
+scripts =
+  runTestSuite
+  slapos
+interpreter=
+  python_for_test
+
+[git-clone-repository]
+recipe = slapos.recipe.build:gitclone
+git-executable = ${git:location}/bin/git
+forbid-download-cache = true
+branch = master
+
+[slapos-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/slapos.git
+
+# XXX we need an unreleased ( 0.4.51 ) version of erp5.util runTestSuite 
+# later we can stop fetching it from git and just use egg
+[erp5.util-repository]
+<= git-clone-repository
+repository = https://lab.nexedi.com/nexedi/erp5.git
+revision = 69013fa0fb67501089c776ab5e75d7bbf2e0e3bc
+
+
+[template]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+mode = 640
+
+[versions]
+# clear the version of tested eggs, to make sure we installed the developped ones
+slapos.test.proftpd =
+erp5.util =
+#erp5.util = 0.4.51

software/proftpd/README.md

+Proftpd with sftp and virtual users
+
+http://www.proftpd.org/docs/
+
+# Features
+
+ * sftp only is enabled
+ * partially uploadloaded are not visible thanks to [`HiddenStores`](http://proftpd.org/docs/directives/linked/config_ref_HiddenStores.html) ( in fact they are, but name starts with `.` )
+ * 5 failed login attempts will cause the host to be temporary banned
+
+
+# TODO
+
+ * only password login is enabled. enabling [`SFTPAuthorizedUserKeys`](http://www.proftpd.org/docs/contrib/mod_sftp.html#SFTPAuthorizedUserKeys) seems to break password only login
+ * log rotation
+ * make sure SFTPLog is useful (seems very verbose and does not contain more than stdout)
+ * make it easier to manage users ( using `mod_auth_web` against an ERP5 endpoint or accepting a list of user/password as instance parameter )
+ * allow configuring webhooks when new file is uploaded
+
+

software/proftpd/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     But avoid directories, they are not portable.
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance-profile]
+filename = instance.cfg.in
+md5sum = 8ed5b4a7940db47ccb386c2f4e3e7273
+
+[instance-default]
+filename = instance-default.cfg.in
+md5sum = 89e145188fe2a223f22f57e5c0c242e4
+
+[proftpd-config-file]
+filename = proftpd-config-file.cfg.in
+md5sum = 46cbe4ea65b445822d3da32c76c7d67d
+

software/proftpd/instance-default.cfg.in

+[buildout]
+parts =
+  promises
+  publish-connection-parameter
+
+
+[instance-parameter]
+# TODO: this is not needed
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+log = ${:var}/log
+srv = ${:home}/srv
+service = ${:etc}/service
+promise = ${:etc}/promise
+proftpd-dir = ${:srv}/proftpd/
+ssh-authorized-keys-dir = ${:etc}/authorized_keys/
+
+[config-file]
+recipe = slapos.recipe.template:jinja2
+template = {{ software_parts_directory }}/${:_buildout_section_name_}/${:_buildout_section_name_}.cfg.in
+rendered = ${directory:etc}/${:_buildout_section_name_}.cfg
+mode = 0644
+extensions = jinja2.ext.do
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[proftpd-userinfo]
+recipe = slapos.cookbook:userinfo
+
+
+[proftpd-password]
+recipe = slapos.cookbook:generate.password
+username = proftpd
+bytes = 12
+
+[proftpd]
+ipv6 = ${instance-parameter:ipv6-random}
+ipv4 = ${instance-parameter:ipv4-random}
+host = ${:ipv6}
+sftp-port = {{ slapparameter_dict.get('port', 8022) }}
+url = sftp://[${:host}]:${:sftp-port}
+data-dir = ${directory:proftpd-dir}
+user=${proftpd-userinfo:pw-name}
+group=${proftpd-userinfo:gr-name}
+sftp-log=${directory:log}/proftpd-sftp.log
+xfer-log=${directory:log}/proftpd-xfer.log
+ban-log=${directory:log}/proftpd-ban.log
+ssh-host-rsa-key=${ssh-host-rsa-key:output}
+ssh-host-dsa-key=${ssh-host-dsa-key:output}
+ssh-host-ecdsa-key=${ssh-host-ecdsa-key:output}
+ssh-authorized-keys-dir = ${directory:ssh-authorized-keys-dir}
+ban-table=${directory:srv}/proftpd-ban-table
+auth-user-file=${auth-user-file:output}
+
+recipe = slapos.cookbook:wrapper
+command-line =
+   {{ proftpd_bin }} --nodaemon --config ${proftpd-config-file:rendered}
+wrapper-path = ${directory:service}/proftpd
+
+[proftpd-listen-promise]
+<= check-port-listening-promise
+hostname = ${proftpd:ipv6}
+port = ${proftpd:sftp-port}
+
+
+[ftpasswd]
+# command line to add a user, invoke with:
+#   ftpasswd --name=bob
+# to prompt for password, or --stdin to read password from stdin
+recipe = slapos.recipe.template:jinja2
+mode = 0700
+extensions = jinja2.ext.do
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+wrapper-path = ${:rendered}
+# XXX: to workaround bug that pw-uid and gr-gir slapos.cookbook:userinfo cannot
+# be used in a buildout part (as they are returned as int and not strings), we
+# use slapos.recipe.template:jinja2 and not a simple slapos.cookbook:wrapper,
+# because only the former is compatible with userinfo.
+# Because we are already in a jinja template, we have to escape jinja template
+# with {% raw %} / {% endraw %}
+template = inline:
+  #!/bin/bash
+  {{ perl_bin }} {{ ftpasswd_bin }} --passwd --home=${proftpd:data-dir} --shell=/bin/false --uid={% raw %}{{ pw_uid }} --gid={{ gr_gid }}{% endraw %} --file ${auth-user-file:output} $@
+context =
+  key pw_uid proftpd-userinfo:pw-uid
+  key gr_gid proftpd-userinfo:gr-gid
+
+[auth-user-file]
+recipe = plone.recipe.command
+output = ${directory:etc}/ftpd.passwd
+command =
+  echo ${proftpd-password:passwd} | ${ftpasswd:wrapper-path} --name=${proftpd-password:username} --stdin
+update-command = ${:command}
+
+
+[ssh-keygen-base]
+recipe = plone.recipe.command
+output = ${directory:etc}/${:_buildout_section_name_}
+command = {{ ssh_keygen_bin }} -f ${:output} -N '' ${:extra-args}
+
+[ssh-host-rsa-key]
+<=ssh-keygen-base
+extra-args=-t rsa
+[ssh-host-dsa-key]
+<=ssh-keygen-base
+extra-args=-t dsa
+[ssh-host-ecdsa-key]
+<=ssh-keygen-base
+extra-args=-t ecdsa -b 521
+
+
+[proftpd-config-file]
+<= config-file
+context =
+  section proftpd proftpd
+  key slapparameter_dict slap-configuration:configuration
+
+
+[promises]
+recipe =
+instance-promises =
+  ${proftpd-listen-promise:path}
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = ${proftpd:url}
+username = ${proftpd-password:username}
+password = ${proftpd-password:passwd}

software/proftpd/instance-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Parameters to instantiate PoFTPd",
+  "additionalProperties": false,
+  "properties": {
+    "port": {
+      "description": "Port number to listen to - default to 8022",
+      "type": "number"
+    }
+  }
+}

software/proftpd/instance-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by ProFTPd instantiation",
+  "additionalProperties": false,
+  "properties": {
+    "url": {
+      "description": "URL of the SFTP service",
+      "pattern": "^sftp://",
+      "type": "string"
+    },
+    "username": {
+      "description": "Default username",
+      "type": "string",
+      "optional": true
+    },
+    "password": {
+      "description": "Password for default username",
+      "type": "string",
+      "optional": true
+    }
+  },
+  "type": "object"
+}

software/proftpd/instance.cfg.in

+[buildout]
+parts = switch-softwaretype
+eggs-directory = {{ buildout['eggs-directory'] }}
+develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[instance-template]
+recipe = slapos.recipe.template:jinja2
+filename = ${:_buildout_section_name_}.cfg
+rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  key slapparameter_dict slap-configuration:configuration
+  raw software_parts_directory {{ buildout['parts-directory'] }}
+  raw proftpd_bin {{ proftpd_bin }}
+  raw ftpasswd_bin {{ ftpasswd_bin }}
+  raw ftpdctl_bin {{ ftpdctl_bin }}
+  raw ssh_keygen_bin {{ ssh_keygen_bin }}
+  raw perl_bin {{ perl_bin }}
+
+[instance-default]
+<= instance-template
+template = {{ instance_default }}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+# XXX Jerome: what is this override for ?
+override = {{ dumps(override_switch_softwaretype |default) }}
+default = instance-default:rendered
+# BBB
+RootSoftwareInstance = ${:default}

software/proftpd/proftpd-config-file.cfg.in

+ServerName "ProFTPD SlapOS"
+ServerType standalone
+MaxInstances  30
+DefaultAddress {{ proftpd['ipv6'] }} {{ proftpd['ipv4']}}
+SocketBindTight on
+DefaultServer off
+Port {{ proftpd['sftp-port'] }}
+
+User {{ proftpd['user'] }}
+Group {{ proftpd['group'] }}
+
+Umask 022
+AllowOverwrite on
+
+# SFTP
+SFTPEngine on
+SFTPHostKey {{ proftpd['ssh-host-rsa-key'] }}
+SFTPHostKey {{ proftpd['ssh-host-dsa-key'] }}
+SFTPHostKey {{ proftpd['ssh-host-ecdsa-key'] }}
+#SFTPAuthorizedUserKeys file:{{ proftpd['ssh-authorized-keys-dir'] }}%u
+
+
+# Logging
+TransferLog {{ proftpd['xfer-log'] }}
+SFTPLog {{ proftpd['sftp-log'] }}
+BanLog {{ proftpd['ban-log'] }}
+
+
+# Virtual users
+RequireValidShell off
+AuthUserFile {{ proftpd['auth-user-file'] }}
+
+
+# Prevent partially uploaded files to be visible
+HiddenStores on
+DeleteAbortedStores on
+AllowStoreRestart off
+
+
+# Limitations of running as non-root user
+WtmpLog off
+# ( we cannot use DefaultRoot ~ )
+
+# Prevent user for escaping their home
+<Directory />
+  <Limit ALL>
+    DenyAll
+  </Limit>
+</Directory>
+
+<Directory {{ proftpd['data-dir'] }}>
+  <Limit ALL>
+    AllowAll
+  </Limit>
+</Directory>
+
+
+
+# Ban failed logins
+MaxLoginAttempts 1
+BanEngine on
+BanTable {{ proftpd['ban-table'] }}
+# 5 failed login attemps in 5 minutes -> ban for 20 minutes
+BanOnEvent MaxLoginAttempts 5/00:05:00 00:20:00 "Too many Failed Login Attempts"
+BanControlsACLs all allow user {{ proftpd['user'] }}
+
+

software/proftpd/software.cfg

+[buildout]
+extends =
+  ../../stack/slapos.cfg
+  ../../component/openssh/buildout.cfg
+  ../../component/proftpd/buildout.cfg
+  buildout.hash.cfg
+
+versions = versions
+parts =
+  slapos-cookbook
+  proftpd-config-file
+  instance-profile
+
+
+[download-file-base]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/${:filename}
+download-only = true
+mode = 0644
+
+[proftpd-config-file]
+<= download-file-base
+
+[instance-default]
+<= download-file-base
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/${:filename}
+rendered = ${buildout:directory}/instance.cfg
+mode = 0644
+extensions = jinja2.ext.do
+context =
+  section buildout buildout
+  key instance_default instance-default:target
+  key proftpd_bin proftpd-output:proftpd
+  key ftpasswd_bin proftpd-output:ftpasswd
+  key ftpdctl_bin proftpd-output:ftpdctl
+  key ssh_keygen_bin openssh-output:keygen
+  key perl_bin proftpd-output:perl
+
+
+[versions]
+collective.recipe.environment = 1.1.0
+collective.recipe.grp = 1.1.0
+slapos.recipe.template = 4.3
+plone.recipe.command = 1.1

software/proftpd/software.cfg.json

+{
+  "name": "ProFTPd",
+  "description": "ProFTPd as a SFTP server with virtual users",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "description": "ProFTPd, with a default user and users added with command line",
+      "request": "instance-input-schema.json",
+      "response": "instance-output-schema.json",
+      "index": 0
+    }
+  }
+}

software/proftpd/test/README.md

+Tests for ProFTPd software release

software/proftpd/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+import glob
+import os
+
+version = '0.0.1.dev0'
+name = 'slapos.test.proftpd'
+long_description = open("README.md").read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' ProFTPd",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'erp5.util',
+        'pysftp',
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/proftpd/test/test.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import shutil
+import urlparse
+import tempfile
+import StringIO
+import subprocess
+
+import pysftp
+from paramiko.ssh_exception import SSHException
+from paramiko.ssh_exception import AuthenticationException
+
+import utils
+
+
+# for development: debugging logs and install Ctrl+C handler
+if os.environ.get('DEBUG'):
+  import logging
+  logging.basicConfig(level=logging.DEBUG)
+  import unittest
+  unittest.installHandler()
+
+
+class ProFTPdTestCase(utils.SlapOSInstanceTestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    return (os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'software.cfg')), )
+
+  def _getConnection(self, username=None, password=None, hostname=None):
+    """Returns a pysftp connection connected to the SFTP
+
+    username and password can be specified and default to the ones from
+    instance connection parameters.
+    another hostname can also be passed.
+    """
+    # this tells paramiko not to verify host key
+    cnopts = pysftp.CnOpts()
+    cnopts.hostkeys = None
+
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse.urlparse(parameter_dict['url'])
+
+    return pysftp.Connection(
+        hostname or sftp_url.hostname,
+        port=sftp_url.port,
+        cnopts=cnopts,
+        username=username or parameter_dict['username'],
+        password=password or parameter_dict['password'])
+
+
+class TestSFTPListen(ProFTPdTestCase):
+  def test_listen_on_ipv4(self):
+    self.assertTrue(self._getConnection(hostname=self.config['ipv4_address']))
+
+  def test_does_not_listen_on_all_ip(self):
+    with self.assertRaises(SSHException):
+      self._getConnection(hostname='0.0.0.0')
+
+
+class TestSFTPOperations(ProFTPdTestCase):
+  """Tests upload / download features we expect in SFTP server.
+  """
+  def setUp(self):
+    self.upload_dir = os.path.join(
+        self.computer_partition_root_path, 'srv', 'proftpd')
+
+  def tearDown(self):
+    for name in os.listdir(self.upload_dir):
+      path = os.path.join(self.upload_dir, name)
+      if os.path.isfile(path) or os.path.islink(path):
+        os.remove(path)
+      else:
+        shutil.rmtree(path)
+
+  def test_simple_sftp_session(self):
+    with self._getConnection() as sftp:
+      # put a file
+      with tempfile.NamedTemporaryFile() as f:
+        f.write("Hello FTP !")
+        f.flush()
+        sftp.put(f.name, remotepath='testfile')
+
+      # it's visible in listdir()
+      self.assertEqual(['testfile'], sftp.listdir())
+
+      # and also in the server filesystem
+      self.assertEqual(['testfile'], os.listdir(self.upload_dir))
+
+      # download the file again, it should have same content
+      tempdir = tempfile.mkdtemp()
+      self.addCleanup(lambda : shutil.rmtree(tempdir))
+      local_file = os.path.join(tempdir, 'testfile')
+      retrieve_same_file = sftp.get('testfile', local_file)
+      with open(local_file) as f:
+        self.assertEqual(f.read(), "Hello FTP !")
+
+  def test_uploaded_file_not_visible_until_fully_uploaded(self):
+    test_self = self
+    class PartialFile(StringIO.StringIO):
+      def read(self, *args):
+        # file is not visible yet
+        test_self.assertNotIn('destination', os.listdir(test_self.upload_dir))
+        # it's just a hidden file
+        test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+        return StringIO.StringIO.read(self, *args)
+
+    with self._getConnection() as sftp:
+      sftp.sftp_client.putfo(PartialFile("content"), "destination")
+
+    # now file is visible
+    self.assertEqual(['destination'], os.listdir(self.upload_dir))
+
+  def test_partial_upload_are_deleted(self):
+    test_self = self
+    with self._getConnection() as sftp:
+      class ErrorFile(StringIO.StringIO):
+        def read(self, *args):
+          # at this point, file is already created on server
+          test_self.assertEqual(['.in.destination.'], os.listdir(test_self.upload_dir))
+          # simulate a connection closed
+          sftp.sftp_client.close()
+          return "something that will not be sent to server"
+      with self.assertRaises(IOError):
+        sftp.sftp_client.putfo(ErrorFile(), "destination")
+    # no half uploaded file is kept
+    self.assertEqual([], os.listdir(self.upload_dir))
+
+  def test_user_cannot_escape_home(self):
+    with self._getConnection() as sftp:
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('..')
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('/')
+      with self.assertRaisesRegexp(IOError, 'Permission denied'):
+        sftp.listdir('/tmp/')
+
+
+class TestUserManagement(ProFTPdTestCase):
+  def test_user_can_be_added_from_script(self):
+    with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+      self._getConnection(username='bob', password='secret')
+
+    subprocess.check_call(
+      'echo secret | %s/bin/ftpasswd --name=bob --stdin' % self.computer_partition_root_path,
+      shell=True)
+    self.assertTrue(self._getConnection(username='bob', password='secret'))
+
+
+class TestBan(ProFTPdTestCase):
+  def test_client_are_banned_after_5_wrong_passwords(self):
+    # Simulate failed 5 login attempts
+    for i in range(5):
+      with self.assertRaisesRegexp(AuthenticationException, 'Authentication failed'):
+        self._getConnection(password='wrong')
+
+    # after that, even with a valid password we cannot connect
+    with self.assertRaisesRegexp(SSHException, 'Connection reset by peer'):
+      self._getConnection()
+
+    # ban event is logged
+    with open(os.path.join(
+        self.computer_partition_root_path, 'var', 'log', 'proftpd-ban.log')) as ban_log_file:
+      self.assertRegexpMatches(
+        ban_log_file.readlines()[-1],
+        'login from host .* denied due to host ban')
+
+
+class TestInstanceParameterPort(ProFTPdTestCase):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    cls.free_port = utils.findFreeTCPPort(cls.config['ipv4_address'])
+    return {'port': cls.free_port}
+
+  def test_instance_parameter_port(self):
+    parameter_dict = self.computer_partition.getConnectionParameterDict()
+    sftp_url = urlparse.urlparse(parameter_dict['url'])
+    self.assertEqual(self.free_port, sftp_url.port)
+    self.assertTrue(self._getConnection())
+
+

software/proftpd/test/utils.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import unittest
+import os
+import socket
+from contextlib import closing
+import logging
+
+from erp5.util.testnode.SlapOSControler import SlapOSControler
+from erp5.util.testnode.ProcessManager import ProcessManager
+import slapos
+
+
+def findFreeTCPPort(ip=''):
+  """Find a free TCP port to listen to.
+  """
+  family = socket.AF_INET6 if ':' in ip else socket.AF_INET
+  with closing(socket.socket(family, socket.SOCK_STREAM)) as s:
+    s.bind((ip, 0))
+    return s.getsockname()[1]
+
+
+class SlapOSInstanceTestCase(unittest.TestCase):
+  @classmethod
+  def getSoftwareURLList(cls):
+    """Return URL of software releases to install.
+
+    To be defined by subclasses.
+    """
+    raise NotImplementedError()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    """Return instance parameters
+
+    To be defined by subclasses if they need to request instance with specific
+    parameters.
+    """
+    return {}
+
+ # TODO: allow subclasses to request a specific software type ?
+
+
+  @classmethod
+  def setUpClass(cls):
+
+    working_directory = os.environ.get(
+        'SLAPOS_TEST_WORKING_DIR',
+        os.path.join(os.path.dirname(__file__), '.slapos'))
+    # To prevent error: Cannot open an HTTP server: socket.error reported
+    # AF_UNIX path too long This `working_directory` should not be too deep.
+    # Socket path is 108 char max on linux
+    # https://github.com/torvalds/linux/blob/3848ec5/net/unix/af_unix.c#L234-L238
+    if len(working_directory + '/inst/supervisord.socket') > 108:
+      raise RuntimeError('working directory too deep, try setting SLAPOS_TEST_WORKING_DIR')
+
+    if not os.path.exists(working_directory):
+      os.mkdir(working_directory)
+
+    cls.config = config = {
+      "working_directory": working_directory,
+      "slapos_directory": working_directory,
+      "log_directory": working_directory,
+      "computer_id": 'slapos.test', # XXX
+      'proxy_database': os.path.join(working_directory, 'proxy.db'),
+      'partition_reference': cls.__name__,
+      # "proper" slapos command must be in $PATH
+      'slapos_binary': 'slapos',
+    }
+
+    # Some tests are expecting that local IP is not set to 127.0.0.1
+    ipv4_address = os.environ.get('LOCAL_IPV4', '127.0.1.1')
+    ipv6_address = os.environ['GLOBAL_IPV6']
+
+    config['proxy_host'] = config['ipv4_address'] = ipv4_address
+    config['ipv6_address'] = ipv6_address
+    config['proxy_port'] = findFreeTCPPort(ipv4_address)
+    config['master_url'] = 'http://{proxy_host}:{proxy_port}'.format(**config)
+
+    cls._process_manager = process_manager = ProcessManager()
+
+    # XXX this code is copied from testnode code
+    slapos_controler = SlapOSControler(
+        working_directory,
+        config
+    )
+
+    slapproxy_log = os.path.join(config['log_directory'], 'slapproxy.log')
+    logger = logging.getLogger(__name__)
+    logger.debug('Configured slapproxy log to %r', slapproxy_log)
+
+    software_url_list = cls.getSoftwareURLList()
+    slapos_controler.initializeSlapOSControler(
+        slapproxy_log=slapproxy_log,
+        process_manager=process_manager,
+        reset_software=False,
+        software_path_list=software_url_list)
+
+    process_manager.supervisord_pid_file = os.path.join(
+           slapos_controler.instance_root, 'var', 'run', 'supervisord.pid')
+
+    software_status_dict = slapos_controler.runSoftwareRelease(config, environment=os.environ)
+    # TODO: log more details in this case
+    assert software_status_dict['status_code'] == 0
+
+    instance_parameter_dict = cls.getInstanceParmeterDict()
+    instance_status_dict = slapos_controler.runComputerPartition(
+        config,
+        cluster_configuration=instance_parameter_dict,
+        environment=os.environ)
+    # TODO: log more details in this case
+    assert instance_status_dict['status_code'] == 0
+
+    # FIXME: similar to test node, only one (root) partition is really supported for now.
+    computer_partition_list = []
+    for i in range(len(software_url_list)):
+      computer_partition_list.append(
+          slapos_controler.slap.registerOpenOrder().request(
+          software_url_list[i],
+          # This is how testnode's SlapOSControler name created partitions
+          partition_reference='testing partition {i}'.format(i=i, **config),
+          partition_parameter_kw=instance_parameter_dict))
+
+    # expose some class attributes so that tests can use them:
+    # the ComputerPartition instances, to getInstanceParmeterDict
+    cls.computer_partition = computer_partition_list[0]
+
+    # the path of the instance on the filesystem, for low level inspection
+    cls.computer_partition_root_path = os.path.join(
+        config['working_directory'],
+        'inst',
+        cls.computer_partition.getId())
+
+
+  @classmethod
+  def tearDownClass(cls):
+    # FIXME: if setUpClass fail, this is not called and leaks zombie processes
+    cls._process_manager.killPreviousRun()
+