Merge branch 'master' into 'master'

Improvements and fixes

See merge request !6

playbook/erp5-standalone.yml

@@ -3,8 +3,10 @@
   connection: local
 
   vars:
-    software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1/software/erp5/software.cfg
+    software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1.1/software/erp5/software.cfg
+    startup_playbook_id: erp5-standalone.yml 
+    playbook_name: erp5
+    request_instance_template: request-erp5-cluster.j2
 
   roles:
-    - erp5-standalone
-
+    - standalone-shared

playbook/install-script.yml

@@ -23,6 +23,9 @@
     - name: generate scripts
       template: src=roles/install-script/templates/dev.j2 dest={{ base_path }}/install/devbook mode=0666
 
+    - name: generate ansible-only
+      template: src=roles/install-script/templates/ansible-only.j2 dest={{ base_path }}/install/ansible mode=0666
+
     - name: generate index.html
       copy: src=roles/install-script/files/index.html dest={{ base_path }}/install/index.html mode=0666
 
@@ -47,3 +50,4 @@
     - { role: "install-script", playbook_yml: "imt-vm-bootstrap.yml", script_path: "install/imt/vm-bootstrap"}
     - { role: "install-script", playbook_yml: "imt-vm-cloudera-manager.yml", script_path: "install/imt/vm-cloudera-manager"}
     - { role: "install-script", playbook_yml: "imt-re6stnet.yml", script_path: "install/imt/re6st"}
+    - { role: "install-script", playbook_yml: "test-suite.yml", script_path: "install/test-suite"}

playbook/re6stnet.yml

@@ -12,6 +12,6 @@
       default: "noname"
 
   roles:
-    - re6stnet-legacy
+    - { role: re6stnet-legacy, package_state: present }
     - { role: package, package_name: ntp, package_state: present }
 

playbook/roles/erp5-standalone/files/erp5-startup

-#!/bin/bash
-# Reruns the ansible playbook, does nothing else
-
-PLAYBOOK_ROOT=/opt/slapos.playbook/playbook/
-PLAYBOOK_FILE=erp5-standalone.yml
-
-cd $PLAYBOOK_ROOT # cd into the playbook directory
-echo "Starting Ansible playbook:"
-
-ansible-playbook $PLAYBOOK_FILE -i  hosts --connection=local

playbook/roles/erp5-standalone/tasks/main.yml

----
-  - name: Supply erp5 software release
-    shell: slapos supply {{ software_release_url }} local_computer
-
-  - name: create partition script
-    copy: src=request-erp5-cluster dest=/tmp/playbook-request-erp5-cluster mode=700
-
-  - name: Request ERP5 Cluster
-    shell: cat /tmp/playbook-request-erp5-cluster | slapos console 
-
-  - name: Add startup script
-    copy: src=erp5-startup dest=/usr/local/bin/erp5-startup mode=755
-  
-  - name: Add to rc.local
-    lineinfile: 
-      dest=/etc/rc.local insertbefore=BOF 
-      line='bash /usr/local/bin/erp5-startup &'
-      state=present
-
-  - name: Get slapos.playbook directory name
-    shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
-    register: tmp_dir
-
-  - name: Check if /opt/slapos.playbook already exists
-    stat: path=/opt/slapos.playbook/
-    register: playbook_state
-
-  - name: Copy slapos.playbook
-    copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
-    when: playbook_state.stat.exists == False

playbook/roles/install-script/files/function-common

@@ -62,10 +62,10 @@ function trueorfalse {
     $xtrace
 }
 
-function isset {
-    [[ -v "$1" ]]
-}
-
+#function isset {
+#    [[ -v "$1" ]]
+#}
+#
 # Control Functions
 # =================
 
@@ -1839,17 +1839,18 @@ function cp_it {
 #
 #     http_proxy=http://proxy.example.com:3128/ no_proxy=repo.example.net ./stack.sh
 
-function export_proxy_variables {
-    if isset http_proxy ; then
-        export http_proxy=$http_proxy
-    fi
-    if isset https_proxy ; then
-        export https_proxy=$https_proxy
-    fi
-    if isset no_proxy ; then
-        export no_proxy=$no_proxy
-    fi
-}
+# Unused function
+#function export_proxy_variables {
+#    if isset http_proxy ; then
+#        export http_proxy=$http_proxy
+#    fi
+#    if isset https_proxy ; then
+#        export https_proxy=$https_proxy
+#    fi
+#    if isset no_proxy ; then
+#        export no_proxy=$no_proxy
+#    fi
+#}
 
 # Returns true if the directory is on a filesystem mounted via NFS.
 function is_nfs_directory {

playbook/roles/install-script/templates/ansible-only.j2

+#!/bin/bash
+
+#### Setup Ansible and load few libraries #####
+BASE_SETUP_SCRIPT_MD5={{ base_setup.stat.md5 }}
+
+wget --no-check-certificate https://deploy.nexedi.cn/base-setup -O /tmp/base-setup
+
+if [ "`md5sum /tmp/base-setup | cut -f1 -d\  `" !=  "$BASE_SETUP_SCRIPT_MD5" ]; then
+  echo "ERROR: base-setup has wrong md5 `md5sum /tmp/base-setup | cut -f1 -d\  ` != $BASE_SETUP_SCRIPT_MD5"
+  exit 1
+fi
+
+source /tmp/base-setup

playbook/roles/install-script/templates/base_setup.j2

 {{ lookup('file', 'roles/install-script/files/function-common') }}
 
 
+export PATH=$PATH:/usr/local/bin:/usr/local/sbin
+
 # Include Additional Functions
 function download_playbook {
     if [ ! -f /etc/opt/slapcache.cfg ]; then
@@ -37,7 +39,7 @@ fi
 
 # Warn users who aren't on an explicitly supported distro, but allow them to
 # override check and attempt installation with ``export FORCE=yes``
-if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|rhel7) ]]; then
+if [[ ! ${DISTRO} =~ (wheezy|jessie|trusty|rhel7|rhel6) ]]; then
     echo "WARNING: this script has not been tested on $DISTRO"
     if [[ "$FORCE" != "yes" ]]; then
         die $LINENO "If you wish to run this script anyway run with FORCE=yes"
@@ -50,7 +52,7 @@ if [[ $DISTRO == "wheezy" ]]; then
 fi
 
 if is_fedora && [[ $DISTRO == "rhel7" ]]; then
-    # RHEL requires EPEL for many Open Stack dependencies
+    # RHEL requires EPEL for many Ansible dependencies
 
     # NOTE: We always remove and install latest -- some environments
     # use snapshot images, and if EPEL version updates they break
@@ -89,6 +91,31 @@ EOF
 
 fi
 
+if is_fedora && [[ $DISTRO == "rhel6" ]]; then
+    if sudo yum repolist enabled epel | grep -q 'epel'; then
+        uninstall_package epel-release || true
+    fi
+
+    cat <<EOF | sudo tee /etc/yum.repos.d/epel-bootstrap.repo
+[epel-bootstrap]
+name=Bootstrap EPEL
+mirrorlist=http://mirrors.fedoraproject.org/mirrorlist?repo=epel-6&arch=\$basearch
+failovermethod=priority
+enabled=0
+gpgcheck=0
+EOF
+    yum-config-manager --enable epel-bootstrap
+    yum_install epel-release || \
+        die $LINENO "Error installing EPEL repo, cannot continue"
+    # EPEL rpm has installed it's version
+    rm -f /etc/yum.repos.d/epel-bootstrap.repo
+
+    is_package_installed yum-utils || install_package yum-utils
+    yum-config-manager --enable rhel-6-server-optional-rpms
+
+    is_package_installed libselinux-python || install_package libselinux-python
+fi
+
 is_package_installed ansible || install_package ansible
 is_package_installed python-setuptools || install_package python-setuptools
 
@@ -97,5 +124,4 @@ if is_ubuntu && [[ $DISTRO == "trusty" ]]; then
   is_package_installed python-pycurl || install_package python-pycurl
 fi
 
-
 ansible localhost -m easy_install -a name=slapcache --connection=local

playbook/roles/ntp/tasks/main.yml

-
-
+---
   - name: Install ntp
     apt: name=ntp state=latest
     when: ansible_os_family == "Debian"
@@ -10,3 +9,8 @@
 
   - name: ensure ntp is runing
     service: name=ntp state=running enabled=yes
+    when: ansible_os_family == "Debian"
+
+  - name: ensure ntp is runing (RedHat)
+    service: name=ntpd state=running enabled=yes
+    when: ansible_os_family == "RedHat"
\ No newline at end of file

playbook/roles/standalone-shared/files/erp5-show

@@ -7,7 +7,8 @@ import glob
 import time
 import getopt
 import sqlite3
-from urllib2 import urlopen
+import ssl
+import urllib2
 from xml.dom import minidom
 import json
 
@@ -20,15 +21,21 @@ def discover_software():
   conn = sqlite3.connect("/opt/slapos/slapproxy.db")
   cur = conn.cursor()
   qry = cur.execute("SELECT DISTINCT software_release FROM partition11")
-  return [row[0] for row in qry]
+  return [row[0] for row in qry if row[0]]
 
 def get_connection_information(software_release):
   conn = sqlite3.connect("/opt/slapos/slapproxy.db")
   cur = conn.cursor()
   qry = cur.execute("SELECT connection_xml FROM partition11 WHERE connection_xml IS NOT NULL AND software_release=?", (software_release,) )
+  xml = None
   for row in qry:
     xml = str(row[0])
     break 
+
+  if xml is None:
+    print software_release
+    return (None, None)
+
   instance = minidom.parseString(xml)
 
   try:
@@ -90,13 +97,51 @@ def status(software_release):
           fmt_date() + ".log for details"
     sys.exit(2)
 
+  ipv6 = None
   # check if the services are actually running (run slapos node and parse output)
   if pw is None:
     zope_ip = "https://" + zope_ip[zope_ip.index("@")+1:]
-  r1 = urlopen(zope_ip)
+  
+  original_zope_ip = zope_ip
+  if "[" in zope_ip and "]" in zope_ip:
+    ipv6 = zope_ip[zope_ip.index("[")+1:zope_ip.index("]")]
+    with open("/etc/hosts", "ra+") as f:
+      if " erp5-instance" not in f.read():
+        f.write("\n%s   erp5-instance\n" % ipv6)
+    zope_ip = zope_ip.replace("[" + ipv6 + "]", "erp5-instance")
+
+  ctx = ssl.create_default_context()
+  ctx.check_hostname = False
+  ctx.verify_mode = ssl.CERT_NONE
+
+  try:
+    r1 = urllib2.urlopen(zope_ip, context=ctx)
+  except urllib2.URLError, e:
+    print "At least one of your services isn't running! Check with slapos node"
+    print "restart a service with slapos node restart slappart:service"
+    print ""
+    print "DEBUG information: %s" % e
+    sys.exit(2)
+  
   if r1.getcode() != 200:
     print "At least one of your services isn't running! Check with slapos node"
     print "restart a service with slapos node restart slappart:service"
+    sys.exit(2)
+
+  if ipv6:
+    print ""
+    print "The URL above may require extra configuration if you want to access it"
+    print "from another machine. You can install an apache locally and include the"
+    print "the follow rewrite rule (http version):"
+    print """ 
+  RewriteRule ^/(.*) %s/VirtualHostBase/http/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
+
+or (https version):
+
+  RewriteRule ^/(.*) %s/VirtualHostBase/https/%%{HTTP_HOST}/VirtualHostRoot/$1 [L,P]
+
+""" % (original_zope_ip, original_zope_ip) 
+
 
 def info(software_release):
   if get_build_status():

playbook/roles/standalone-shared/tasks/main.yml

 ---
+  - name: create partition script
+    template: src={{ request_instance_template }} dest=/tmp/playbook-request-{{ playbook_name }} mode=700
+
+  - name: Supply and Request ERP5 Cluster
+    shell: cat /tmp/playbook-request-{{ playbook_name }} | slapos console
+
   - name: create erp5-show
     copy: src=erp5-show dest=/usr/local/bin/erp5-show mode=755
 
+  - name: Add startup script
+    template: src=start-script.j2 dest=/usr/local/bin/{{ playbook_name }}-startup mode=755
+
+  - name: Add to rc.local
+    lineinfile:
+      dest=/etc/rc.local insertbefore=BOF
+      line='bash /usr/local/bin/{{ playbook_name }}-startup &'
+      state=present
+
+  - name: Get slapos.playbook directory name
+    shell: cd /tmp/tmpplaybookerp5-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
+    register: tmp_dir
+
+  - name: Check if /opt/slapos.playbook already exists
+    stat: path=/opt/slapos.playbook/
+    register: playbook_state
+
+  - name: Copy slapos.playbook
+    copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
+    when: playbook_state.stat.exists == False

playbook/roles/erp5-standalone/files/request-erp5-cluster → playbook/roles/standalone-shared/templates/request-erp5.j2

 import json
-software_url = 'https://lab.nexedi.cn/nexedi/slapos/raw/erp5-cluster-0.1/software/erp5/software.cfg' 
+
+software_url = '{{ software_release_url }}'
 # Choose a SlapOS Node
 # If you are deploying SlapOS Master with Webrunner, then computer_id is 'slaprunner'
 computer_id = 'local_computer'
 
+supply(software_url, computer_id)
+
 parameter_dict = {
   "timezone": "UTC",
   "site-id": "erp5",
@@ -33,7 +36,7 @@ parameter_dict = {
 }
 
 # Choose a title
-title = "instance-of-erp5-cluster"
+title = "instance-of-{{ playbook_name }}"
 
 request(software_url,
   title,

playbook/roles/wendelin-standalone/files/request-wendelin → playbook/roles/standalone-shared/templates/request-wendelin.j2

-import json
-# copied from the request-erp5-cluster script, only changed the url to point to wendelin
-software_url = 'https://lab.nexedi.cn/nexedi/slapos/raw/1.0/software/wendelin/software.cfg'
-# Choose a SlapOS Node
-# If you are deploying SlapOS Master with Webrunner, then computer_id is 'slaprunner'
+software_url = '{{ software_release_url }}'
 computer_id = 'local_computer'
 
-parameter_dict = {
-}
-
 # Choose a title
-title = "instance-of-wendelin"
+title = "instance-of-{{ playbook_name }}"
+
+supply(software_url, computer_id)
 
 request(software_url,
   title,

playbook/roles/standalone-shared/templates/start-script.j2

+#!/bin/bash
+# Reruns the ansible playbook, does nothing else
+
+cd /opt/slapos.playbook/playbook/
+
+ansible-playbook {{ startup_playbook_id }}  -i  hosts --connection=local

playbook/roles/switch-to-playbook/tasks/main.yml

+---
+  - include: remove-slapos-package.yml

playbook/roles/switch-to-playbook/tasks/remove-slapos-package.yml

+- name: a play that runs entirely on the ansible host
+  hosts: 127.0.0.1
+  connection: local
+
+  tasks:
+
+    - name: Remove slapos.package setup and configuration
+      shell: rm -rf /usr/local/lib/python2.*/dist-packages/slapos.package*
+    
+    - name: Remove slapos.package setup and configuration
+      file: path={{ item }} state=absent
+      with_items:
+        - /usr/local/bin/slappkg-discover
+        - /usr/local/bin/slappkg-conf
+        - /usr/local/bin/slappkg-update
+        - /usr/local/bin/slappkg-update-raw
+        - /usr/local/bin/slappkg-upload-key
+        - /etc/opt/update.cfg
+         

playbook/roles/test-suite/files/log-server

+#!/usr/bin/python2.7
+import socket
+import errno
+import cgi
+import os
+from BaseHTTPServer import BaseHTTPRequestHandler
+from BaseHTTPServer import HTTPServer
+from SimpleHTTPServer import SimpleHTTPRequestHandler
+
+class MyHandler(BaseHTTPRequestHandler):
+
+  def do_GET(self):
+    if self.path == '/ip':
+      self.send_response(200)
+      self.send_header('Content-type', 'text/html')
+      self.end_headers()
+      self.wfile.write('Your IP address is %s' % self.client_address[0]) 
+      return    
+    else:
+      return SimpleHTTPRequestHandler.do_GET(self)
+
+  def do_POST(self):     
+    form = cgi.FieldStorage(
+      fp=self.rfile,
+      headers=self.headers,
+      environ={"REQUEST_METHOD": "POST"}
+    )
+
+    lw.write_log(form.list[0].value, form.list[1:])
+    self.send_response(200)
+
+class HTTPServerV6(HTTPServer):
+  address_family = socket.AF_INET6
+
+class logWriter():
+  def write_log(self, fp, contents):
+    print os.path.dirname(fp)
+    try:
+      os.makedirs(os.path.dirname(fp))
+    except OSError as exception:
+      if exception.errno != errno.EEXIST:
+        raise
+    try:
+      with open(fp, 'a') as myfile:
+        for l in contents:
+          myfile.write(l.value)
+    except IOError as e:
+      print "Unable to open file" #Does not exist OR no read permissions
+      with open(fp, 'w') as myfile:
+        for l in contents:
+          myfile.write(l.value)
+    print "Finished writing"
+
+if __name__ == '__main__':  
+  lw = logWriter()
+  server = HTTPServerV6(('::', 9000), MyHandler)
+  print 'Starting server, use <Ctrl-C> to stop'
+  server.serve_forever()

playbook/roles/test-suite/files/log-uploader

+#!/usr/bin/python2.7
+
+import os
+import sys
+import time
+from glob import glob
+import requests
+from watchdog.observers import Observer
+from watchdog.events import FileSystemEventHandler
+
+class MyHandler(FileSystemEventHandler):
+  def on_created(self, event):
+    if ".completed" in event.file:
+      print "partition completed!"
+      upload_file(event.file, fmt_date() + ": partition completed")
+
+def fmt_date():
+  return time.strftime("[ %Y%m%d %H:%M:%S ]")
+
+def dir_date():
+  return time.strftime("%Y%m%d/")
+
+class uploader():
+  url = 'http://[2001:67c:1254:e:a9::bafb]:9000/post'
+  # takes a file name and contents to append. Generates server file_path (where it should be saved on server)
+  def upload_file(self, file_name, contents):
+    values = {'file_path': dir_date() + file_name, 'contents': contents}
+    r = requests.post(self.url, data=values)
+
+def check_startup():
+  pid = str(os.getpid())
+  pidfile = "/tmp/slaplogger.pid"
+
+  if os.path.isfile(pidfile):
+    print "%s already exists, exiting" % pidfile
+    sys.exit()
+  else:
+    file(pidfile, 'w').write(pid)
+  os.unlink(pidfile)
+
+def scan_files(files):
+  try:  
+    with open("/opt/slapos/lines", 'r') as myfile:
+      software_line = myfile.readlines()
+  except:
+    software_line = [0] * len(files)
+  with open("/opt/slapos/lines", 'w'):
+    pass  
+  place = 0
+
+  for f in files:
+    print software_line[place]
+    print software_line
+    with open(f, 'r') as myfile:
+      new_lines = myfile.readlines()[software_line[place]:]
+    with open("/opt/slapos/lines", 'a') as myfile:
+      myfile.write(str(software_line[place]) + str(len(new_lines)))
+    print "Uploading " + f + "..."
+    c.upload_file(os.path.basename(f), new_lines)
+    print f + " finished uploading!"
+    place += 1
+  print "Files uploaded"
+
+
+if __name__ == "__main__":
+  check_startup()
+  c = uploader()
+  files = ["/opt/slapos/log/slapos-node-software.log", "/opt/slapos/log/slapos-node-format.log", "/opt/slapos/log/slapos-node-instance.log"]
+
+  # first check if .completed exists in any partitions
+  paths = ["/opt/slapgrid/slappart" + p + ".completed" for p in glob('/opt/slapgrid/*')]
+  for p in paths: 
+    try:
+      f = open(p, 'r')
+      upload_file(p, fmt_date() + ": partition completed")    
+    except IOError as e:
+      pass
+
+  event_handler = MyHandler()
+  observer = Observer()
+
+
+  # check if .completed exists or not, and write it to the server log
+  observer.schedule(event_handler, path='/opt/slapgrid/', recursive=True)
+  observer.start()
+
+  try:
+    while True:
+      scan.files(files)
+  except KeyboardInterrupt:
+    observer.stop()
+  observer.join()

playbook/roles/test-suite/tasks/main.yml

+---
+  - name: download software_release name
+    get_url: url=https://lab.nexedi.cn/krruzic/test-repository/raw/master/software
+             dest=/tmp/playbook_software_name
+
+  - name: create log-uploader
+    copy: src=log-uploader dest=/usr/local/bin/log-uploader mode=755
+
+  - name: add log-uploader to cron
+    cron: name="Ensure log-uploader is running" 
+          minute="*/6"
+          job="usr/local/bin/log-uploader"
+
+  - name: run ansible playbook
+    shell: ansible-playbook "{{ lookup('file', '/tmp/playbook_software_name') }}" -i hosts
+
+  # install dependencies
+  - easy_install: name=pip
+  - pip: name=requests
+  - pip: name=watchdog

playbook/roles/vm-bootstrap/tasks/hostname.yml

@@ -41,7 +41,7 @@
     when: hostname_file.stat.exists == True
 
   - name: setting cluster 
-    get_url: url="{{ lookup('file', '/tmp/cluster.hash') }}/hosts" dest=/tmp/hosts mode=666 validate_certs=no
+    shell: wget --no-check-certificate "{{ lookup('file', '/tmp/cluster.hash') }}/hosts" -O /tmp/hosts
     when: cluster_hash.stat.exists == True
 
   - name: stat /tmp/hosts

playbook/roles/wendelin-standalone/files/wendelin-startup

-#!/bin/bash
-# Reruns the ansible playbook, does nothing else
-
-PLAYBOOK_ROOT=/opt/slapos.playbook/
-PLAYBOOK_FILE=wendelin-standalone.yml
-
-cd $PLAYBOOK_ROOT # cd into the playbook directory
-
-ansible-playbook $PLAYBOOK_FILE -i  hosts --connection=local

playbook/roles/wendelin-standalone/meta/main.yml

----
-dependencies:
-  - slapos-proxy

playbook/roles/wendelin-standalone/tasks/main.yml

----
-  - name: Supply wendelin software release
-    shell: slapos supply {{ software_release_url }} local_computer
-
-  - name: create partition script
-    copy: src=request-wendelin dest=/tmp/playbook-request-wendelin mode=700
-
-  - name: Request Wendelin instance
-    shell: cat /tmp/playbook-request-wendelin | slapos console 
-
-  - name: Add startup script
-    copy: src=wendelin-startup dest=/usr/local/bin/wendelin-startup mode=755
-
-  - name: Add to rc.local
-    lineinfile: 
-      dest=/etc/rc.local insertbefore=BOF 
-      line='bash /usr/local/bin/wendelin-startup &'
-      state=present
-
-  - name: Get slapos.playbook directory name
-    shell: cd /tmp/tmpplaybookwendelin-standalone.*/slapos.playbook.git/playbook/; echo $(pwd)/
-    register: tmp_dir
-
-  - name: Check if /opt/slapos.playbook already exists
-    stat: path=/opt/slapos.playbook/
-    register: playbook_state
-
-  - name: Copy slapos.playbook
-    copy: src={{ tmp_dir.stdout }} dest=/opt/slapos.playbook/
-    when: playbook_state.stat.exists == False

playbook/test-suite.yml

+- name: a play that runs entirely on the ansible host
+  hosts: 127.0.0.1
+  connection: local
+
+  roles:
+    - test-suite

playbook/wendelin-standalone.yml

@@ -3,8 +3,10 @@
   connection: local
 
   vars:
-    software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/1.0/software/wendelin/software.cfg
+    software_release_url: https://lab.nexedi.cn/nexedi/slapos/raw/1.0.3/software/wendelin/software.cfg
+    startup_playbook_id: wendelin-standalone.yml
+    playbook_name: wendelin
+    request_instance_template: request-wendelin.j2
 
   roles:
-    - wendelin-standalone
     - standalone-shared