Remove git server and add slapos format to work on webrunner

See merge request !747

software/slaprunner/README.rst

@@ -139,17 +139,6 @@ Example :
 	<parameter id="custom-frontend-basic-auth">true</parameter>
 	</instance>
 
-Git repositories :
-------------------
-
-It is easy to give access to your git repository/ies to everyone, or to clone it on your own computer. For this, fetch the git_url parameter returned by your webrunner instance. It should look like that : https://[IPV6]:PORT/git/YourRepo.git/
-
-To create the repo, go to the folder srv/runner/project and initiate a new git repo (git init/clone --bare YourRepo). Then, to clone the repo, just append its name to the previous URL.
-
-Please notice that your repositories are in public access for reading. However, an account is needed to push to the repository (use your webrunner account, or create a new user in the parameters section of the webrunner).
-
-For the moment, the PORT is the port of monitoring, which is 9685.
-
 Things to notice for the nex developer :
 ----------------------------------------
 

software/slaprunner/buildout.hash.cfg

@@ -18,11 +18,11 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
 
 [template-runner]
 filename = instance-runner.cfg
-md5sum = 7100c6678576c285322f1445029cd4f2
+md5sum = 00915c7ad17e335c87be40bae4bd0cfb
 
 [template-runner-import-script]
 filename = template/runner-import.sh.jinja2
-md5sum = 2852689640b632fc713d7a603ce66207
+md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
 
 [instance-runner-import]
 filename = instance-runner-import.cfg.in
@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
 
 [template-resilient]
 filename = instance-resilient.cfg.jinja2
-md5sum = 105ed7f54f251c64a2d34559360a5604
+md5sum = bd0ad0b80d2b39189f9665c48f1b3830
 
 [template_nginx_conf]
 filename = nginx_conf.in
@@ -42,7 +42,7 @@ md5sum = 2b06f7eb9a1d45d250d4b92a944db925
 
 [template_httpd_conf]
 filename = httpd_conf.in
-md5sum = 40825ef2d259ae3b86dfd2f28f597d3a
+md5sum = 1d96c29b931383b2a87b9ae049d07d74
 
 [template_launcher]
 filename = launcher.in
@@ -50,7 +50,11 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
 
 [template-slapos-cfg]
 filename = template/slapos.cfg.in
-md5sum = e6a3ca1604ae5458248135cd6de0f3e6
+md5sum = 95de0677e78fc06cc8304cc6caea9169
+
+[template-slapformat-definition.cfg]
+filename = template/slapformat-definition.cfg.in
+md5sum = a94299dd2133956060486609e30b97ec
 
 [template-parameters]
 filename = parameters.xml.in
@@ -62,7 +66,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
 
 [template-supervisord]
 filename = template/supervisord.conf.in
-md5sum = 28f69b57c2835bddfcfbe312563eef51
+md5sum = 37f053d75752e998fc3bb9e4bf29d776
 
 [template-listener-slapgrid]
 filename = template/listener_slapgrid.py.in

software/slaprunner/httpd_conf.in

@@ -96,33 +96,3 @@ Alias /share {{ parameters.runner_home }}
     </LimitExcept>
 </Directory>
 
-SetEnv GIT_HTTP_EXPORT_ALL
-ScriptSock {{ parameters.cgid_sock }}
-ScriptAlias /git/ {{ parameters.git_http_backend }}/
-ScriptAlias /git-public/ {{ parameters.git_http_backend }}/
-
-RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
-RewriteCond %{REQUEST_URI} /git-receive-pack$
-
-<LocationMatch "^/git/">
-        SetEnv GIT_PROJECT_ROOT {{ parameters.project_private_folder }}
-        Order Deny,Allow
-        Deny from env=AUTHREQUIRED
-
-        AuthType Basic
-        AuthName "Git Access"
-        AuthUserFile "{{ parameters.htpasswd_file }}"
-        Require valid-user
-</LocationMatch>
-
-<LocationMatch "^/git-public/">
-        SetEnv GIT_PROJECT_ROOT {{ parameters.project_public_folder }}
-        Order Deny,Allow
-        Deny from env=AUTHREQUIRED
-
-        AuthType Basic
-        AuthName "Git Access"
-        AuthUserFile "{{ parameters.htpasswd_file }}"
-        Require valid-user
-        Satisfy any
-</LocationMatch>

software/slaprunner/instance-resilient.cfg.jinja2

@@ -12,7 +12,7 @@
   {% set number_of_instances = 2 %}
 {% endif %}
 
-{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url', 'git-public-url', 'git-private-url'] -%}
+{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url'] -%}
 {% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
 {% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
 {% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
@@ -72,8 +72,6 @@ init-password = ${publish-early:init-password}
 ssh-command = ${request-runner:connection-ssh-command}
 webdav-url = ${request-runner:connection-webdav-url}
 public-url = ${request-runner:connection-public-url}
-git-public-url = ${request-runner:connection-git-public-url}
-git-private-url = ${request-runner:connection-git-private-url}
 {% if slapparameter_dict.get('custom-frontend-backend-url') -%}
 custom-frontend-url = ${request-runner:connection-custom-frontend-url}
 {% endif %}

software/slaprunner/instance-runner-output-schema.json

@@ -29,14 +29,6 @@
     "public_url": {
       "description": "Url of the public folder of your runner",
       "type": "string"
-    },
-    "git_public_url": {
-      "description": "Public url of your repositories.",
-      "type": "string"
-    },
-    "git_private_url": {
-      "description": "Private url for your repositories. You can use it to push.",
-      "type": "string"
     }
   },
   "type": "object"

software/slaprunner/instance-runner.cfg

@@ -21,6 +21,7 @@ common-runner-parts =
   shellinabox
   shellinabox-service
   slapos-cfg
+  slapformat-definition.cfg
   cron-entry-prepare-software
   deploy-instance-parameters
   instance-software
@@ -151,8 +152,6 @@ project-test = $${:test}/project
 software-test = $${:test}/software
 instance-test = $${:test}/instance
 sessions = $${buildout:directory}/.sessions
-private-project = $${:home}/.git-private
-public-project = $${:home}/.git-public
 
 [slaprunner]
 slaprunner = ${buildout:bin-directory}/slaprunner
@@ -162,6 +161,7 @@ supervisor = ${buildout:bin-directory}/slapgrid-supervisorctl
 git-binary = ${git:location}/bin/git
 root_check = false
 slapos.cfg = $${directory:etc}/slapos.cfg
+slapformat-definition.cfg = $${directory:etc}/slapformat-definition.cfg
 working-directory = $${runnerdirectory:home}
 project-directory = $${runnerdirectory:project}
 instance_root = $${runnerdirectory:instance-root}
@@ -389,11 +389,7 @@ htpasswd_file = $${directory:etc}/.htpasswd
 etc_dir = $${directory:etc}
 var_dir = $${directory:var}
 project_folder = $${directory:project}
-project_private_folder = $${runnerdirectory:private-project}
-project_public_folder = $${runnerdirectory:public-project}
 runner_home = $${runnerdirectory:home}
-git_http_backend = ${git:location}/libexec/git-core/git-http-backend
-cgid_sock = $${directory:run}/cgid.sock
 httpd_cors_file = $${slaprunner-httpd-cors:location}
 
 [httpd-conf]
@@ -591,8 +587,6 @@ init-user = $${runner-htpasswd:user}
 init-password = $${runner-htpasswd:password}
 ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
 ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshd-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
-git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
-git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
 monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
 {% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' -%}
 url =  https://$${request-frontend:connection-domain}
@@ -672,6 +666,17 @@ mode = 700
 context =
   section slaprunner test-runner
 
+[slapformat-definition.cfg]
+recipe = slapos.recipe.template:jinja2
+template = ${template-slapformat-definition.cfg:location}/${template-slapformat-definition.cfg:filename}
+rendered = $${slaprunner:slapformat-definition.cfg}
+mode = 700
+context =
+  section slaprunner slaprunner
+  key partition_user user-info:pw-name
+  import codecs codecs
+  raw buildout_shared_part_list_dump $${slaprunner:buildout-shared-part-list-dump}
+
 [shellinabox]
 recipe = slapos.recipe.template:jinja2
 # We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
@@ -785,6 +790,10 @@ slapproxy-autostart = true
 slapproxy-startsecs = 1
 slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
 slapproxy-log = $${directory:log}/slapproxy.log
+slapformat = slapformat
+slapformat-command = $${slaprunner:slapos} node format --cfg $${:slapos-cfg} --verbose --logfile $${:slapformat-log} --now -i $${:slapformat-definition.cfg}
+slapformat-log = $${directory:log}/slapos-node-format.log
+slapformat-startretries = 0
 socket_name = unix://$${:socket_path}
 socket_path = $${directory:tmp}/supervisord.sock
 startsecs = 0
@@ -792,6 +801,7 @@ startsecs = 0
 # Processes should handle their logs by themselves
 stderr_logfile = $${directory:log}/supervisord-errors.log
 slapos-cfg = $${slaprunner:slapos.cfg}
+slapformat-definition.cfg = $${slaprunner:slapformat-definition.cfg}
 
 [supervisord-conf]
 recipe = slapos.recipe.template:jinja2

software/slaprunner/software.cfg

@@ -42,6 +42,7 @@ common-parts =
   instance-runner-import
   instance-runner-export
   template-slapos-cfg
+  template-slapformat-definition.cfg
   template-slapuser-script
 # XXX: we have to manually add this for resilience
   rdiff-backup
@@ -110,6 +111,10 @@ recipe = slapos.recipe.build:download
 < = template-download-base
 filename = slapos.cfg.in
 
+[template-slapformat-definition.cfg]
+< = template-download-base
+filename = slapformat-definition.cfg.in
+
 [template-parameters]
 < = download-only-base
 

software/slaprunner/template/runner-import.sh.jinja2

@@ -173,7 +173,7 @@ if 1:
 
   # We want to replace the last occurence only
   old_software_release, partition = sys.argv[1], sys.argv[2]
-  for match in re.finditer("(slappart|test0-)[0-9][0-9]*", old_software_release):
+  for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
     start, end = match.start(), match.end()
 
   print old_software_release[:start] + partition + old_software_release[end:]

software/slaprunner/template/slapformat-definition.cfg.in

+[computer]
+# Address example: 2001:67c:1254:27::1/ffff:ffff:ffff:ffff::/64 
+address = {{ slaprunner['ipv6'] }}/64
+
+{% for partition_index in range(int(slaprunner['partition-amount'])) %}
+[partition_{{ partition_index }}]
+address =  {{ slaprunner['ipv4'] }}/255.255.255.255 {{ slaprunner['ipv6'] }}/64
+user = {{ partition_user }}
+pathname = slappart{{ partition_index }}
+network_interface =
+
+{% endfor %}
+

software/slaprunner/template/slapos.cfg.in

@@ -22,6 +22,11 @@ pidfile_instance =  {{slaprunner['pidfile-instance']}}
 
 [slapformat]
 partition_amount = {{ slaprunner['partition-amount'] }}
+alter_user = false
+alter_network = false
+create_tab = false
+create_tun = false
+computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
 
 [slaprunner]
 slapos = {{ slaprunner['slapos'] }}

software/slaprunner/template/supervisord.conf.in

@@ -44,6 +44,20 @@ stderr_logfile = {{ supervisord['no_logfile'] }}
 directory = {{ supervisord['directory'] }}
 environment = PATH="{{- supervisord['path'] -}}"
 
+[program:{{- supervisord['slapformat'] -}}]
+command = {{ supervisord['slapformat-command'] }}
+process_name = {{ supervisord['slapformat'] }}
+numprocs = {{ supervisord['numprocs'] }}
+autostart = {{ supervisord['autostart'] }}
+exitcodes = {{ supervisord['exitcodes'] }}
+startretries = {{ supervisord['slapformat-startretries'] }}
+startsecs = {{ supervisord['startsecs'] }}
+autorestart = {{ supervisord['autorestart'] }}
+stdout_logfile = {{ supervisord['no_logfile'] }}
+stderr_logfile = {{ supervisord['no_logfile'] }}
+directory = {{ supervisord['directory'] }}
+environment = PATH="{{- supervisord['path'] -}}"
+
 [program:{{- supervisord['slapproxy'] -}}]
 command = {{ supervisord['slapproxy-command'] }}
 process_name = {{ supervisord['slapproxy'] }}

software/slaprunner/test/test.py

@@ -49,7 +49,6 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
         os.path.join(os.path.dirname(__file__), '..',
                      'software%s.cfg' % ("-py3" if six.PY3 else ""))))
 
-
 class SlaprunnerTestCase(SlapOSInstanceTestCase):
   # Slaprunner uses unix sockets, so it needs short paths.
   __partition_reference__ = 's'
@@ -106,26 +105,6 @@ class TestWeb(SlaprunnerTestCase):
     self.assertEqual(requests.codes.ok, hello.status_code)
     self.assertIn('<b>Hello</b>', hello.text)
 
-  # git seems broken, these are 404 now...
-  @unittest.expectedFailure
-  def test_git_private(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    url = parameter_dict['git-private']
-    resp = requests.get(url, verify=False)
-    self.assertEqual(requests.codes.unauthorized, resp.status_code)
-    resp = requests.get(
-        url,
-        verify=False,
-        auth=(parameter_dict['init-user'], parameter_dict['init-password']))
-    self.assertEqual(requests.codes.ok, resp.status_code)
-
-  @unittest.expectedFailure
-  def test_git_public(self):
-    parameter_dict = self.computer_partition.getConnectionParameterDict()
-    url = parameter_dict['git-public']
-    resp = requests.get(url, verify=False)
-    self.assertEqual(requests.codes.ok, resp.status_code)
-
 
 class TestSSH(SlaprunnerTestCase):
   @classmethod
@@ -275,6 +254,7 @@ class ServicesTestCase(SlaprunnerTestCase):
 
 class TestInstanceResilient(SlaprunnerTestCase):
   instance_max_retry = 20
+
   @classmethod
   def getInstanceSoftwareType(cls):
     return 'resilient'
@@ -287,8 +267,6 @@ class TestInstanceResilient(SlaprunnerTestCase):
         'backend-url',
         'feed-url-runner-1-pull',
         'feed-url-runner-1-push',
-        'git-private-url',
-        'git-public-url',
         'init-password',
         'init-user',
         'monitor-base-url',

stack/resilient/buildout.hash.cfg

@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
 
 [resilient-web-takeover-cgi-script-download]
 filename = resilient-web-takeover-cgi-script.py.in
-md5sum = 7a0f1be7e9d05d0c68748322abe1f511
+md5sum = 4137b7ac6b20a6cd897ad58f51a763b1
 
 [template-wrapper]
 filename = templates/wrapper.in

stack/resilient/resilient-web-takeover-cgi-script.py.in

@@ -48,7 +48,7 @@ def getLatestBackupDate():
   if not db_keys:
     result = False
   else:
-    last_backup = db[db_keys[0]]
+    last_backup = float(db[db_keys[0]])
     for callback in db_keys:
       timestamp = float(db[callback])
       if timestamp > last_backup: