Merge branch 'slaprunner-paas' into slaprunner-layout

Conflicts:
	slapos/runner/static/css/styles.css
	slapos/runner/static/js/scripts/process.js
	slapos/runner/static/js/scripts/softwareFolder.js
	slapos/runner/static/js/scripts/viewlog.js
	slapos/runner/templates/account.html
	slapos/runner/templates/layout.html
	slapos/runner/templates/softwareFolder.html
	slapos/runner/templates/viewLog.html
	slapos/runner/views.py

CHANGES.txt

+0.39 (2014-02-20)
+-----------------
+
+ * Slaprunner: new web interface design
+ * Slaprunner: one function handle both "run software" and "run instance" [9c660c0]
+ * Slaprunner: building and deploying can be customized [0db1f6b, b33bd1f]
+ * Slaprunner: adds a multi-user feature [efad6d]
+ * Slaprunner: add fullscreen mode for text edition [83d1dc]
+ * Slaprunner: direct access to monitoring of running instance, if it exists [f8e7bf3]
+
+0.38.1 (2013-12-06)
+-------------------
+
+ * Slaprunner: do not delete proxy.db on each run software [71777fc0]
+
+0.38 (2013-12-03)
+-----------------
+
+ * Slaprunner: adds an integrated shell [ca6a670a]
+ * Slaprunner: uses basic authentification [05913751]
+ * Slaprunner: adds automated deployment of a Software Release [c8ab1273]
+ * Slaprunner: flask development server replaced by Gunicorn, a WSGI server [48d60d0f]
+ * Slaprunner: new test scenario for auto-deployment [c6007954]
+ * Runner resiliencytestsuite: adds basic auth support [3c03f12b]
+ * Runner resiliencytestsuite: tests can be done on only one Slapos node [07198d87]
+ 
+0.37.4 (2013-10-15)
+-------------------
+
+ * Improve QEMU QMP wrapper by adding drive-backup method and other helpers. [0afb7d6, 95d0c8b]
+
+0.37.3 (2013-10-10)
+-------------------
+
+* pubsub: don't swallow output of subprocess to allow debug. [c503484]
+
+0.37.2 (2013-10-10)
+-------------------
+
+ * Add QEMU QMP wrapper. [9e819a8]
+ * KVM resiliency test: update docstring about how to setup disk image. [dbe347f]
+ * KVM resiliency test: change key for each clone. [7ef1db3]
+
 0.37.1 (2013-10-03)
 -------------------
 

setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '0.37.1'
+version = '0.39'
 name = 'slapos.toolbox'
 long_description = open("README.txt").read() + "\n" + \
     open("CHANGES.txt").read() + "\n"
@@ -70,6 +70,7 @@ setup(name=name,
           'onetimeupload = slapos.onetimeupload:main',
           'pubsubnotifier = slapos.pubsub.notifier:main',
           'pubsubserver = slapos.pubsub:main',
+          'qemu-qmp-client = slapos.qemuqmpclient:main',
           'shacache = slapos.shacache:main',
           'slapbuilder = slapos.builder:main',
           'slapcontainer = slapos.container:main',

slapos/pubsub/__init__.py

-from datetime import datetime
+import argparse
 import csv
 import feedparser
+import httplib # To avoid magic numbers
 import io
-import socket
 import json
-import time
 import math
-import httplib # To avoid magic numbers
-import argparse
 import os
+import socket
+import sys
+import time
+from datetime import datetime
 from hashlib import sha512
 
 from atomize import Entry
@@ -19,6 +20,9 @@ from flask import abort
 from flask import request
 app = Flask(__name__)
 
+# csv entries can be very large, increase limit.
+csv.field_size_limit(sys.maxsize)
+
 @app.route('/get/<feed>')
 def get_feed(feed):
   global app

slapos/pubsub/notifier.py

@@ -4,7 +4,6 @@
 import argparse
 import csv
 import httplib
-import os
 import socket
 import subprocess
 import sys
@@ -32,28 +31,23 @@ def main():
 
   args = parser.parse_args()
 
-  with open(os.devnull) as devnull:
-    command = subprocess.Popen(args.executable[0],
-                               stdin=subprocess.PIPE,
-                               stdout=devnull,
-                               stderr=subprocess.PIPE,
-                               close_fds=True)
-    command.stdin.flush()
-    command.stdin.close()
-
-    command_failed = (command.wait() != 0)
-    command_stderr = command.stderr.read()
-
-    if command_failed:
-      content = ("<p>Failed with returncode <em>%d</em>.</p>"
-                 "<p>Standard error output is :</p><pre>%s</pre>") % (
-        command.poll(),
-        command_stderr.replace('&', '&amp;')\
-                      .replace('<', '&lt;')\
-                      .replace('>', '&gt;'),
-      )
-    else:
-      content = "<p>Everything went well.</p>"
+  try:
+    content = subprocess.check_output(
+        args.executable[0],
+        stderr=subprocess.STDOUT
+    )
+    exit_code = 0
+  except subprocess.CalledProcessError as e:
+    content = e.output
+    exit_code = e.returncode
+
+  print content
+
+  content += ("\n<p>Failed with returncode <em>%d</em>.</p>"
+              "<p>Output is: </p><pre>%s</pre>" % (
+      exit_code,
+      content.replace('&', '&amp;').replace('<', '&lt;').replace('>', '&gt;')
+  ))
 
   with open(args.logfile[0], 'a') as file_:
     cvsfile = csv.writer(file_)
@@ -64,9 +58,8 @@ def main():
       'slapos:%s' % uuid.uuid4(),
     ])
 
-  if command_failed:
-    sys.stderr.write('%s\n' % command_stderr)
-    sys.exit(1)
+  if  exit_code != 0:
+    sys.exit(exit_code)
 
   print 'Fetching %s feed...' % args.feed_url[0]
 

slapos/qemuqmpclient/__init__.py

+##############################################################################
+#
+# Copyright (c) 2013 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import argparse
+import json
+import os
+import pprint
+import socket
+import time
+
+def parseArgument():
+  """
+  Very basic argument parser. Might blow up for anything else than
+  "./executable mysocket.sock stop/resume".
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--suspend', action='store_const', dest='action', const='suspend')
+  parser.add_argument('--resume', action='store_const', dest='action', const='resume')
+  parser.add_argument('--create-snapshot', action='store_const', dest='action', const='createSnapshot')
+  parser.add_argument('--create-internal-snapshot', action='store_const', dest='action', const='createInternalSnapshot')
+  parser.add_argument('--delete-internal-snapshot', action='store_const', dest='action', const='deleteInternalSnapshot')
+  parser.add_argument('--drive-backup', action='store_const', dest='action', const='driveBackup') 
+  parser.add_argument('--query-commands', action='store_const', dest='action', const='queryCommands')
+
+  parser.add_argument('--socket', dest='unix_socket_location', required=True)
+  parser.add_argument('remainding_argument_list', nargs=argparse.REMAINDER)
+  args = parser.parse_args()
+  return args.unix_socket_location, args.action, args.remainding_argument_list
+
+
+class QemuQMPWrapper(object):
+  """
+  Small wrapper around Qemu's QMP to control a qemu VM.
+  See http://git.qemu.org/?p=qemu.git;a=blob;f=qmp-commands.hx for
+  QMP API definition.
+  """
+  def __init__(self, unix_socket_location):
+    self.socket = self.connectToQemu(unix_socket_location)
+    self.capabilities()
+
+  @staticmethod
+  def connectToQemu(unix_socket_location):
+    """
+    Create a socket, connect to qemu, be sure it answers, return socket.
+    """
+    if not os.path.exists(unix_socket_location):
+      raise Exception('unix socket %s does not exist.' % unix_socket_location)
+
+    print 'Connecting to qemu...'
+    so = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    connected = False
+    while not connected:
+      try:
+        so.connect(unix_socket_location)
+      except socket.error:
+        time.sleep(1)
+        print 'Could not connect, retrying...'
+      else:
+        connected = True
+    so.recv(1024)
+
+    return so
+
+
+  def _send(self, message):
+    self.socket.send(json.dumps(message))
+    data = self.socket.recv(65535)
+    try:
+      return json.loads(data)
+    except ValueError:
+      print 'Wrong data: %s' % data
+
+  def _getVMStatus(self):
+    response = self._send({'execute': 'query-status'})
+    if response:
+      return self._send({'execute': 'query-status'})['return']['status']
+    else:
+      raise IOError('Empty answer')
+
+  def _waitForVMStatus(self, wanted_status):
+    while True:
+      try:
+        actual_status = self._getVMStatus()
+        if actual_status == wanted_status:
+          return
+        else:
+          print 'VM in %s status, wanting it to be %s, retrying...' % (
+              actual_status, wanted_status)
+          time.sleep(1)
+      except IOError:
+          print 'VM not ready, retrying...'
+
+
+  def capabilities(self):
+    print 'Asking for capabilities...'
+    self._send({'execute': 'qmp_capabilities'})
+
+  def suspend(self):
+    print 'Suspending VM...'
+    self._send({'execute': 'stop'})
+    self._waitForVMStatus('paused')
+
+  def resume(self):
+    print 'Resuming VM...'
+    self._send({'execute': 'cont'})
+    self._waitForVMStatus('running')
+
+  def _queryBlockJobs(self, device):
+    return self._send({'execute': 'query-block-jobs'})
+
+  def _getRunningJobList(self, device):
+    result = self._queryBlockJobs(device)
+    if result.get('return'):
+      return result['return']
+    else:
+      return
+
+  def driveBackup(self, backup_target, source_device='virtio0', sync_type='full'):
+    print 'Asking Qemu to perform backup to %s' % backup_target
+    # XXX: check for error
+    self._send({
+        'execute': 'drive-backup',
+        'arguments': {
+            'device': source_device,
+            'sync': sync_type,
+            'target': backup_target,
+         }
+    })
+    while self._getRunningJobList(backup_target):
+      print 'Job is not finished yet.'
+      time.sleep(20)
+
+  def createSnapshot(self, snapshot_file, device='virtio0'):
+    print self._send({
+        'execute': 'blockdev-snapshot-sync',
+        'arguments': {
+            'device': device,
+            'snapshot-file': snapshot_file,
+         }
+    })
+
+  def createInternalSnapshot(self, name=None, device='virtio0'):
+    if name is None:
+      name = int(time.time())
+    self._send({
+        'execute': 'blockdev-snapshot-internal-sync',
+        'arguments': {
+            'device': device,
+            'name': name,
+         }
+    })
+
+  def deleteInternalSnapshot(self, name, device='virtio0'):
+    self._send({
+        'execute': 'blockdev-snapshot-delete-internal-sync',
+        'arguments': {
+            'device': device,
+            'name': name,
+         }
+    })
+
+  def queryCommands(self):
+    pprint.pprint(self._send({'execute': 'query-commands'})['return'])
+
+def main():
+  unix_socket_location, action, remainding_argument_list = parseArgument()
+  qemu_wrapper = QemuQMPWrapper(unix_socket_location)
+
+  if remainding_argument_list:
+    getattr(qemu_wrapper, action)(*remainding_argument_list)
+  else:
+    getattr(qemu_wrapper, action)()
+
+if __name__ == '__main__':
+  main()
+

slapos/resiliencytest/README.txt

@@ -3,10 +3,13 @@ service (usually deployed using "test" software type).
 
 One entry point, if the service has been deployed
 from a "scalability" test node, so with special parameters, will automatically
-communicate to an ERP5 TestNode Master, and start the test.
+communicate when the instance is started to an ERP5 TestNode Master, and start the test.
 
-The other entry point is supposed to be run manually from a simple "test"
-instance without special parameter, and will manually run the test.
+The other entry point, "bin/runStandaloneTest", is supposed to be run manually from a simple "test"
+instance without special parameter (just request an instance of your software
+release using the dedicated test software-type made for the occasion).
+This is quite useful if you simply want to run the resiliency tests without having the whole
+dedicated test infrastructure.
 
 
 
@@ -44,3 +47,17 @@ For reference: How-to deploy the whole test system
 Note: the slapos nodes are currently deployed using slapos-in-partition.
 Note: you have to manually kill -10 the erp5testnode process to start deployment of test because it doesn't know when SR installation is finished.
 Note: you have to manually run slapos-node-software --all on the slapos nodes if you are developping the SR you are testing.
+
+------------
+STANDALONE TESTS
+
+Here is an example on how to deploy standalone tests on the webrunner, which means without using erp5.
+
+1/ Deploy a SlapRunner software instance using the type test.
+2/ In slapos.org, you should tell on which server you want to deploy your instances. You can adapt to your case the parameter.xml above. For the first time, you can deploy all the instances on the same node, it will run the tests faster, and it will be easier to debug :
+	<?xml version='1.0' encoding='utf-8'?>
+	<instance>
+	  <parameter id="_">{"cluster": {"-sla-0-computer_guid": "COMP-XXXX", "-sla-1-computer_guid": "COMP-XXXX", "-sla-2-computer_guid": "COMP-XXXX"}}</parameter>
+	</instance>
+3/ Then go to the root instance folder : it is the one who has only "runStandaloneResiliencyTestSuite" in its bin folder.
+4/ Run ./bin/runStandaloneResiliencyTestSuite and wait :) it would return "success" or "failure"

slapos/resiliencytest/suites/kvm.py

@@ -109,7 +109,9 @@ def runTestSuite(server_url, key_file, cert_file,
   3/ Resilience is done, wait XX seconds
   4/ For each clone: do a takeover. Check that IPv6 of new main instance is different. Check, when doing a http request to the new VM that will fetch the stored random number, that the sent number is the same.
 
-  Note: disk image is a simple debian with the following python code running at boot:
+  Note: disk image is a simple debian with gunicorn and flask installed:
+    apt-get install python-setuptools; easy_install gunicorn flask
+  With the following python code running at boot in /root/number.py:
 
   import os
   
@@ -119,7 +121,7 @@ def runTestSuite(server_url, key_file, cert_file,
   storage = 'storage.txt'
   
   @app.route("/")
-  def greeting_list(): # 'cause they are several greetings, and plural is forbidden.
+  def greeting_list(): # 'cause there are several greetings, and plural is forbidden.
     return "Hello World"
   
   @app.route("/get")
@@ -128,13 +130,39 @@ def runTestSuite(server_url, key_file, cert_file,
   
   @app.route("/set")
   def set():
-    if os.path.exists(storage):
-      abort(503)
+    #if os.path.exists(storage):
+    #  abort(503)
     open(storage, 'w').write(request.args['key'])
     return "OK"
   
   if __name__ == "__main__":
     app.run(host='0.0.0.0', port=80)
+
+
+  Then create the boot script:
+  echo "cd /root; /usr/local/bin/gunicorn number:app -b 0.0.0.0:80 -D --error-logfile /root/error_log --access-logfile /root/access_log" > /etc/init.d/gunicorn-number
+  chmod +x /etc/init.d/gunicorn-number
+  update-rc.d gunicorn-number defaults
+
+
+  There also is a script that randomly generates I/O in /root/io.sh:
+
+  #!/bin/sh
+  # Randomly generates high I/O on disk. Goal is to write on disk so that
+  # it flushes at the same time that snapshot of disk image is done, to check if
+  # it doesn't corrupt image.
+  # Ayayo!
+  while [ 1 ]; do
+    dd if=/dev/urandom of=random count=2k
+    sync
+    sleep 0.2
+  done
+
+  Then create the boot script:
+  echo "/bin/sh /root/io.sh &" > /etc/init.d/io
+  chmod +x /etc/init.d/io
+  update-rc.d io defaults
+
   """
   slap = slapos.slap.slap()
   slap.initializeConnection(server_url, key_file, cert_file)
@@ -146,9 +174,6 @@ def runTestSuite(server_url, key_file, cert_file,
   ip = fetchMainInstanceIP(partition, software, kvm_rootinstance_name)
   logger.info('KVM IP is %s.' % ip)
 
-  key = setRandomKey(ip)
-  logger.info('Key set for test in current KVM: %s.' % key)
-
   # In resilient stack, main instance (example with KVM) is named "kvm0",
   # clones are named "kvm1", "kvm2", ...
   clone_count = int(total_instance_count) - 1
@@ -158,6 +183,10 @@ def runTestSuite(server_url, key_file, cert_file,
   # Test each clone
   while current_clone <= clone_count:
     logger.info('Testing kvm%s.' % current_clone)
+
+    key = setRandomKey(ip)
+    logger.info('Key set for test in current KVM: %s.' % key)
+
     logger.info('Sleeping for %s seconds.' % SLEEP_TIME)
     time.sleep(SLEEP_TIME)
 

slapos/resiliencytest/suites/resiliencytestsuite.py

@@ -32,6 +32,7 @@ import slapos.slap
 
 import logging
 import time
+import os
 
 class ResiliencyTestSuite(object):
   """
@@ -104,6 +105,14 @@ class ResiliencyTestSuite(object):
     """
     raise NotImplementedError('Overload me, I am an abstract method.')
 
+  def deleteTimestamp():
+    """
+    XXX-Nicolas delete .timestamp in test partition to force the full processing
+    by slapgrid, to force the good parameters to be passed to the instances of the tree
+    """
+    home = os.getenv('HOME')
+    timestamp = os.path.join(home, '.timestamp')
+    os.remove(timestamp)
 
   def _getPartitionParameterDict(self):
     """
@@ -115,6 +124,7 @@ class ResiliencyTestSuite(object):
         software_type='resilient',
         partition_reference=self.root_instance_name
     ).getConnectionParameterDict()
+    self.deleteTimestamp()
 
   def _returnNewInstanceParameter(self, parameter_key, old_parameter_value):
     """
@@ -126,8 +136,8 @@ class ResiliencyTestSuite(object):
     new_parameter_value = None
     while not new_parameter_value or new_parameter_value == 'None' or  new_parameter_value == old_parameter_value:
       self.logger.info('Not ready yet. SlapOS says new parameter value is %s' % new_parameter_value)
-      time.sleep(60)
       new_parameter_value = self._getPartitionParameterDict().get(parameter_key, None)
+      time.sleep(120)
     self.logger.info('New parameter value of instance is %s' % new_parameter_value)
 
     return new_parameter_value

slapos/resiliencytest/suites/slaprunner.py

@@ -28,7 +28,10 @@
 
 from .resiliencytestsuite import ResiliencyTestSuite
 
+import base64
 import cookielib
+import json
+from lxml import etree
 import random
 import string
 import time
@@ -63,6 +66,7 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
         300
     )
 
+
   def _connectToSlaprunner(self, resource, data=None):
     """
     Utility.
@@ -81,41 +85,49 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
 
   def _login(self):
     self.logger.debug('Logging in...')
-    self._connectToSlaprunner('doLogin', data='clogin=%s&cpwd=%s' % (
-        self.slaprunner_user,
-        self.slaprunner_password)
-    )
+    b64string = base64.encodestring('%s:%s' % (self.slaprunner_user, self.slaprunner_password))[:-1]
+    self._opener_director.addheaders = [('Authorization', 'Basic %s'%b64string)]
 
   def _retrieveInstanceLogFile(self):
     """
     Store the logfile (=data) of the instance, check it is not empty nor it is
     html.
     """
+    time.sleep(30)
     data = self._connectToSlaprunner(
-        resource='fileBrowser',
-        data='opt=9&filename=log.log&dir=instance_root%252Fslappart0%252Fvar%252Flog%252F'
+        resource='getFileContent',
+        data="file=instance_root/slappart0/var/log/log.log"
     )
-    self.logger.info('Retrieved data are:\n%s' % data)
-
-    if data.find('<') is not -1:
-      raise IOError(
-          'Could not retrieve logfile content: retrieved content is html.'
-      )
-    if data.find('Could not load') is not -1:
-      raise IOError(
-          'Could not retrieve logfile content: server could not load the file.'
-      )
-    if data.find('Hello') is -1:
-      raise IOError(
-          'Could not retrieve logfile content: retrieve content does not match "Hello".'
-      )
+    try:
+        data = json.loads(data)['result']
+        self.logger.info('Retrieved data are:\n%s' % data)
+    except (ValueError, KeyError):
+        if data.find('<') is not -1:
+          raise IOError(
+              'Could not retrieve logfile content: retrieved content is html.'
+          )
+        if data.find('Could not load') is not -1:
+          raise IOError(
+              'Could not retrieve logfile content: server could not load the file.'
+          )
+        if data.find('Hello') is -1:
+          raise IOError(
+              'Could not retrieve logfile content: retrieve content does not match "Hello".'
+          )
     return data
 
   def _waitForSoftwareBuild(self):
-    while self._connectToSlaprunner(resource='slapgridResult', data='position=0&log=').find('"software": true') is not -1:
-      self.logger.info('Software release is still building. Sleeping...')
-      time.sleep(15)
-    self.logger.info('Software Release has been built / is no longer building.')
+    #while self._connectToSlaprunner(resource='slapgridResult', data='position=0&log=').find('"software": true') is not -1:
+    #  self.logger.info('Software release is still building. Sleeping...')
+    #  time.sleep(15)
+    #self.logger.info('Software Release has been built / is no longer building.')
+    try:
+      while self._connectToSlaprunner(resource='isSRReady') != "1":
+         self.logger.info('Software release is still building. Sleeping...')
+         time.sleep(15)
+    except (NotHttpOkException, urllib2.HTTPError):
+      # The nginx frontend might timeout before software release is finished.
+      self._waitForSoftwareBuild()
 
   def _buildSoftwareRelease(self):
     self.logger.info('Building the Software Release...')
@@ -158,6 +170,20 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
         data='path=workspace/slapos/software/%s/' % software_name
     )
 
+  def _getRcode(self):
+    #XXX-Nicolas: hardcoded url. Best way right now to automate the tests...
+    monitor_url = self.monitor_url + "?script=zero-knowledge%2Fsettings.cgi"
+    result = self._opener_director.open(monitor_url, 
+                                       "password=passwordtochange")
+
+    if result.getcode() is not 200:
+      raise NotHttpOkException(result.getcode())
+
+    page = result.read().strip()
+    html = etree.HTML(page)
+
+    input = html.xpath("//input[@name='recovery-code']")
+    return input[0].get('value')
 
   def generateData(self):
     self.slaprunner_password = ''.join(
@@ -180,7 +206,8 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
     parameter_dict = self._getPartitionParameterDict()
     self.slaprunner_backend_url = parameter_dict['backend_url']
     self.logger.info('backend_url is %s.' % self.slaprunner_backend_url)
-    slaprunner_recovery_code = parameter_dict['password_recovery_code']
+    self.monitor_url = parameter_dict['monitor_url']
+    slaprunner_recovery_code = self._getRcode()
 
     self.logger.debug('Creating the slaprunner account...')
     self._connectToSlaprunner(
@@ -199,6 +226,7 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
     self._openSoftwareRelease('helloworld')
 
     self._buildSoftwareRelease()
+    time.sleep(15)
     self._deployInstance()
 
     self.data = self._retrieveInstanceLogFile()
@@ -219,9 +247,7 @@ class SlaprunnerTestSuite(ResiliencyTestSuite):
     )
     self._login()
     self._waitForSoftwareBuild()
-    # XXX: in theory, it should be done automatically by slaprunner.
-    #      In practice, it is still too dangerous for ERP5 instances.
-    self._deployInstance()
+    time.sleep(15)
     new_data = self._retrieveInstanceLogFile()
 
     if new_data == self.data:

slapos/runner/__init__.py

@@ -6,52 +6,14 @@ import ConfigParser
 import datetime
 import logging
 import logging.handlers
-from optparse import OptionParser, Option
 import os
-import slapos.runner.process
+from slapos.htpasswd import HtpasswdFile
+from slapos.runner.process import setHandler
 import sys
 from slapos.runner.utils import runInstanceWithLock
+from slapos.runner.views import *
 
-
-class Parser(OptionParser):
-  """
-  Parse all arguments.
-  """
-  def __init__(self, usage=None, version=None):
-    """
-    Initialize all possible options.
-    """
-    option_list = [
-      Option("-l", "--log_file",
-             help="The path to the log file used by the script.",
-             type=str),
-      Option("-v", "--verbose",
-             default=False,
-             action="store_true",
-             help="Verbose output."),
-      Option("-c", "--console",
-             default=False,
-             action="store_true",
-             help="Console output."),
-      Option("-d", "--debug",
-             default=False,
-             action="store_true",
-             help="Debug mode."),
-    ]
-
-    OptionParser.__init__(self, usage=usage, version=version,
-                          option_list=option_list)
-
-  def check_args(self):
-    """
-    Check arguments
-    """
-    (options, args) = self.parse_args()
-    if len(args) != 1:
-      self.error("Incorrect number of arguments")
-
-    return options, args[0]
-
+TRUE_VALUES = (1, '1', True, 'true', 'True')
 
 class Config:
   def __init__(self):
@@ -61,22 +23,18 @@ class Config:
     self.logger = None
     self.verbose = None
 
-  def setConfig(self, option_dict, configuration_file_path):
+  def setConfig(self):
     """
     Set options given by parameters.
     """
-    self.configuration_file_path = os.path.abspath(configuration_file_path)
-    # Set options parameters
-    for option, value in option_dict.__dict__.items():
-      setattr(self, option, value)
+    self.configuration_file_path = os.path.abspath(os.getenv('RUNNER_CONFIG'))
 
     # Load configuration file
     configuration_parser = ConfigParser.SafeConfigParser()
-    configuration_parser.read(configuration_file_path)
-    # Merges the arguments and configuration
+    configuration_parser.read(self.configuration_file_path)
 
     for section in ("slaprunner", "slapos", "slapproxy", "slapformat",
-                    "sshkeys_authority", "gitclient", "cloud9_IDE"):
+                    "sshkeys_authority", "gitclient"):
       configuration_dict = dict(configuration_parser.items(section))
       for key in configuration_dict:
         if not getattr(self, key, None):
@@ -88,17 +46,17 @@ class Config:
     if self.console:
       self.logger.addHandler(logging.StreamHandler())
 
-    if self.log_file:
-      if not os.path.isdir(os.path.dirname(self.log_file)):
-        # fallback to console only if directory for logs does not exists and
-        # continue to run
-        raise ValueError('Please create directory %r to store %r log file' % (
-          os.path.dirname(self.log_file), self.log_file))
-      else:
-        file_handler = logging.FileHandler(self.log_file)
-        file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
-        self.logger.addHandler(file_handler)
-        self.logger.info('Configured logging to file %r' % self.log_file)
+    self.log_file = self.log_dir + '/slaprunner.log'
+    if not os.path.isdir(os.path.dirname(self.log_file)):
+    # fallback to console only if directory for logs does not exists and
+    # continue to run
+      raise ValueError('Please create directory %r to store %r log file' % (
+      os.path.dirname(self.log_file), self.log_file))
+    else:
+      file_handler = logging.FileHandler(self.log_file)
+      file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
+      self.logger.addHandler(file_handler)
+      self.logger.info('Configured logging to file %r' % self.log_file)
 
     self.logger.info("Started.")
     self.logger.info(os.environ['PATH'])
@@ -107,30 +65,46 @@ class Config:
       self.logger.debug("Verbose mode enabled.")
 
 
+def checkHtpasswd(config):
+  """XXX:set for backward compatiblity
+  create a htpassword if etc/.users exist"""
+  user = os.path.join(config['etc_dir'], '.users')
+  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
+  if os.path.exists(user) and not os.path.exists(htpasswdfile):
+    data = open(user).read().strip().split(';')
+    htpasswd = HtpasswdFile(htpasswdfile, create=True)
+    htpasswd.update(data[0], data[1])
+    htpasswd.save()
+  else:
+    return
+
+def checkJSONConfig(config):
+  """create a default json file with some parameters inside
+  if the file has never been created"""
+  json_file = os.path.join(config['etc_dir'], 'config.json')
+  if not os.path.exists(json_file):
+    params = {
+      'run_instance' : True,
+      'run_software' : True,
+      'max_run_instance' : 3,
+      'max_run_software' : 2
+    }
+    open(json_file, "w").write(json.dumps(params))
+    
+
 def run():
   "Run default configuration."
-  usage = "usage: %s [options] CONFIGURATION_FILE" % sys.argv[0]
-
-  try:
-    # Parse arguments
-    config = Config()
-    config.setConfig(*Parser(usage=usage).check_args())
+  # Parse arguments
+  config = Config()
+  config.setConfig()
 
-    if os.getuid() == 0:
-        # avoid mistakes (mainly in development mode)
-        raise Exception('Do not run SlapRunner as root.')
-
-    serve(config)
-    return_code = 0
-  except SystemExit as err:
-    # Catch exception raise by optparse
-    return_code = err
-
-  sys.exit(return_code)
+  if os.getuid() == 0:
+    # avoid mistakes (mainly in development mode)
+    raise Exception('Do not run SlapRunner as root.')
 
+  serve(config)
 
 def serve(config):
-  from views import app
   from werkzeug.contrib.fixers import ProxyFix
   workdir = os.path.join(config.runner_workdir, 'project')
   software_link = os.path.join(config.runner_workdir, 'softwareLink')
@@ -145,14 +119,17 @@ def serve(config):
     SECRET_KEY=os.urandom(24),
     PERMANENT_SESSION_LIFETIME=datetime.timedelta(days=31),
   )
+  checkHtpasswd(app.config)
+  checkJSONConfig(app.config)
   if not os.path.exists(workdir):
     os.mkdir(workdir)
   if not os.path.exists(software_link):
     os.mkdir(software_link)
-  slapos.runner.process.setHandler()
+  setHandler()
   config.logger.info('Running slapgrid...')
-  runInstanceWithLock(app.config)
+  if app.config['auto_deploy_instance'] in TRUE_VALUES:
+    runInstanceWithLock(app.config)
   config.logger.info('Done.')
   app.wsgi_app = ProxyFix(app.wsgi_app)
-  app.run(host=config.runner_host, port=int(config.runner_port),
-      debug=config.debug, threaded=True)
+
+run()

slapos/runner/gittools.py

@@ -121,36 +121,40 @@ def getDiff(project):
     result = safeResult(str(e))
   return result
 
+def gitCommit(project, msg):
+  """Commit changes for the specified repository
+  Args:
+    project: directory of the local repository
+    msg: commit message"""
+  code = 0
+  json = ""
+  repo = Repo(project)
+  if repo.is_dirty:
+    git = repo.git
+    #add file to be commited
+    files = repo.untracked_files
+    for f in files:
+      git.add(f)
+    #Commit all modified and untracked files
+    git.commit('-a', '-m', msg)
+  else:
+    code = 1
+    json = "Nothing to be commited"
+  return jsonify(code=code, result=json)
 
-def gitPush(project, msg):
-  """Commit and Push changes for the specified repository
+def gitPush(project):
+  """Push changes for the specified repository
   Args:
     project: directory of the local repository
     msg: commit message"""
   code = 0
   json = ""
-  undo_commit = False
   try:
-    repo = Repo(project)
-    if repo.is_dirty:
-      git = repo.git
-      current_branch = repo.active_branch.name
-      #add file to be commited
-      files = repo.untracked_files
-      for f in files:
-        git.add(f)
-      #Commit all modified and untracked files
-      git.commit('-a', '-m', msg)
-      undo_commit = True
-      #push changes to repo
-      git.push('origin', current_branch)
-      code = 1
-    else:
-      json = "Nothing to commit"
-      code = 1
+    #push changes to repo
+    current_branch = repo.active_branch.name
+    git.push('origin', current_branch)
+    code = 1
   except Exception as e:
-    if undo_commit:
-      git.reset("HEAD~")  # undo previous commit
     json = safeResult(str(e))
   return jsonify(code=code, result=json)
 

slapos/runner/static/css/styles.css

@@ -48,7 +48,7 @@ th{
 }
 table.small th{padding: 4px;font-size: 16px;}
 textarea {
-	width:932px;
+  width:932px;
   font-family: 'Arial,Helvetica Neue',Tahoma,Helvetica,sans-serif;
 }
 
@@ -257,6 +257,10 @@ input[type="radio"], input[type="checkbox"]{
 	margin-bottom:10px;
 }
 
+#commitmsg {
+	width:95%;
+}
+
 .message {
 	color:#FF5500;
 	line-height:21px;
@@ -406,7 +410,7 @@ padding: 10px;height: 80px;padding-bottom:15px;}
 }
 #code{
     float: right;
-    width: 692px;
+    width: 680px;
 }
 #details_head{margin-bottom: 10px;}
 
@@ -629,6 +633,26 @@ a.no-right-border:focus{border-right:none}
 a.lshare img{
     margin: 5px;
 }
+.box_header{
+  background: #E4E4E4;
+  width: 100%;
+  height: 30px;
+  padding-top: 2px;
+  text-indent: 5px;
+  color: #737373;
+  text-shadow: 0px 1px #F1F1F1;
+  border-bottom: 3px solid #6DB9DD;
+}
+.box_header li{float: left;border: 1px solid #AAB8C2;padding: 1px 5px 1px 0;margin-left: 5px;}
+.box_header li:last-child{border:none}
+.box_header li > span{cursor: pointer; height: 22px; display: block;line-height: 22px;font-weight: bold;padding: 1px;}
+.box_header li:hover{border: 1px solid #57A1D6;background: #C7C7C7;}
+.box_header li:last-child:hover{background:none; border: none}
+.box_header li>a{font-weight: bold; font-size:15px;display: block;padding: 2px;}
+.save_btn{background: url(../images/icon_save.png) center right no-repeat;width: 60px;}
+.swith_btn{background: url(../images/gnome-session-switch.png) center right no-repeat;width: 105px;}
+.flist_btn{background: url(../images/list2_down.png) center right no-repeat;width: 26px;}
+.fmenu_btn{background: url(../images/ui_menu_blue.png) center right no-repeat;width: 58px;}
 
 #editor, #editorViewer {
     margin: 0;
@@ -772,4 +796,24 @@ padding:10px; font-size:14px; color:#03406A}
 .ace_search{
   width: 350px;
   max-width: 350px;
-}
\ No newline at end of file
+}
+
+.fullScreen .fullScreen-editor{
+  height: auto!important;
+  width: auto!important;
+  border: 0;
+  margin: 0;
+  position: fixed !important;
+  top: 0;
+  bottom: 0;
+  left: 0;
+  right: 0;
+  z-index: 10000;
+}
+
+.fullScreen {
+  overflow: hidden;
+}
+
+/*  *******************  */
+#shellinabox{width:100%; min-height:530px;}

slapos/runner/static/js/scripts/account.js

@@ -45,6 +45,8 @@ $(document).ready(function () {
             return false;
         }
         send = true;
+	var base_url = 'https://' + $("input#username").val()  + ':'
+	    + $("input#password").val() + '@' + location.host
         $.ajax({
             type: "POST",
             url: $SCRIPT_ROOT + ((hasAccount) ? '/updateAccount' : '/configAccount'),
@@ -57,7 +59,8 @@ $(document).ready(function () {
             },
             success: function (data) {
                 if (data.code === 1) {
-                    window.location.href = $SCRIPT_ROOT + "/";
+                    url = 'https://' + $("input#username").val() + ':' + $("input#password").val() + '@' + location.host + $SCRIPT_ROOT + '/';
+                    window.location.href = url;
                 } else {
                     $("#error").Popup(data.result, {type: 'error', duration: 5000});
                 }
@@ -67,4 +70,68 @@ $(document).ready(function () {
         });
         return false;
     });
+    $("#save").click(function () {
+        if (send) {
+            return false;
+        }
+        send = true;
+        $.ajax({
+            type: "POST",
+            url: $SCRIPT_ROOT + '/updateBuildAndRunConfig',
+            data: {
+                run_instance: $("input#run_instance").is(':checked'),
+                run_software: $("input#run_software").is(':checked'),
+                max_run_instance: $("input#max_run_instance").val(),
+                max_run_software: $("input#max_run_software").val()
+            },
+            success: function (data) {
+                if (data.code === 1) {
+                    $("#error").Popup(data.result, {type: 'alert', duration: 5000});
+                } else {
+                    $("#error").Popup(data.result, {type: 'error', duration: 5000});
+                }
+                send = false;
+            },
+            error: function () { send = false; }
+        });
+        return false;
+    });
+    $("#add_user").click(function () {
+        if ($("input#new_username").val() === "" || !$("input#new_username").val().match(/^[\w\d\._\-]+$/)) {
+            $("#error").Popup("Invalid user name. Please check it!", {type: 'alert', duration: 3000});
+            return false;
+        }
+        if (!$("input#new_rcode").val().match(/^[\w\d]+$/)) {
+            $("#error").Popup("Please enter your password recovery code.", {type: 'alert', duration: 3000});
+            return false;
+        }
+        if (send) {
+            return false;
+        }
+        send = true;
+        $.ajax({
+            type: "POST",
+            url: $SCRIPT_ROOT + '/addUser',
+            data: {
+                username: $("input#new_username").val(),
+                password: $("input#new_password").val(),
+  		rcode: $("input#new_rcode").val(),
+            },
+            success: function (data) {
+                if (data.code === 1) {
+                    $("#error").Popup(data.result, {type: 'info', duration: 5000});
+                } else if (data.code === 0) {
+                    $("#error").Popup(data.result, {type: 'error', duration: 5000});
+		} else {
+                    $("#error").Popup(data.result, {type: 'alert', duration: 5000});
+                }
+                send = false;
+                $("input#new_username").val('');
+                $("input#new_password").val('');
+  		$("input#new_rcode").val('');
+            },
+            error: function () { send = false; }
+        });
+        return false;
+    });
 });

slapos/runner/static/js/scripts/common.js

@@ -104,3 +104,4 @@ function bindRemove() {
         }
     });
 }(jQuery, document, this));
+

slapos/runner/static/js/scripts/login.js

@@ -27,7 +27,9 @@ $(document).ready(function () {
         $("#login").removeClass("button").addClass("dsblebutton");
         $.post(url, param, function (data) {
             if (data.code === 1) {
-                window.location.href = $SCRIPT_ROOT + '/';
+	        url = 'https://' + param.clogin + ':' + param.cpwd + '@'
+		+ location.host + $SCRIPT_ROOT + '/';
+                window.location.href = url;
             } else {
                 $("#error").Popup(data.result, {type: 'alert', duration: 3000});
             }

slapos/runner/static/js/scripts/process.js

@@ -89,12 +89,6 @@ function getRunningState() {
             }
         }).error(function () {
             clearAll(false);
-        }).complete(function () {
-            if (running) {
-                setTimeout(function () {
-                    getRunningState();
-                }, speed);
-            }
         });
 }
 
@@ -147,7 +141,8 @@ function bindRun() {
         } else {
             if (!isRunning()) {
                 setCookie("slapgridCMD", "Instance");
-                window.location.href = $SCRIPT_ROOT + "/viewLog";
+                if (window.location.pathname === "/viewLog")
+                    window.location.href = $SCRIPT_ROOT + "/viewLog";
             }
         }
         return false;
@@ -173,6 +168,13 @@ function updateStatus(elt, val) {
       $(src).children('p').text("Processing");
       break;
   }
+  // in case of failure
+  if ($("#salpgridLog").text().indexOf("Failed to run buildout profile") !== -1) {
+    var src = '#' + elt + '_run_state', value = 'state_' + "stopped";
+    $(src).removeClass();
+    $(src).addClass(value);
+    $(src).children('p').text("Buildout Failed");
+  }
 }
 
 function setRunningState(data) {
@@ -197,6 +199,13 @@ function setRunningState(data) {
                   $("#softrun").addClass('slapos_stop');
                   $("#running img").before('<p id="running_info" class="instance">Running instance...</p>');
                 }
+            		if (processType === "Software") {
+                    running = false;
+                    $("#running_info").remove();
+                    $("#softrun").addClass('slapos_run');
+                    $("#softrun").removeClass('slapos_stop');
+                    $("#instrun").click();
+            		}
                 processType = "Instance";
             }
         }
@@ -215,6 +224,7 @@ function setRunningState(data) {
           $("#slapswitch").text('Access application');
         }
         $("#running").hide();
+        $("#running_info").remove();
         running = false; //nothing is currently running
         $("#softrun").removeClass('slapos_stop');
         $("#softrun").addClass('slapos_run');
@@ -238,11 +248,14 @@ function runProcess(urlfor, data) {
         if ( $("#running_info").children('span').length > 0 ) {
           $("#running_info").children('p').remove();
         }
-        setRunningState(data);
-        setTimeout(getRunningState, 6000);
     }
 }
 
+setInterval('GetStateRegularly()', 800);
+function GetStateRegularly() {
+    getRunningState();
+}
+
 function checkSavedCmd() {
     "use strict";
     var result = getCookie("slapgridCMD");

slapos/runner/static/js/scripts/repo.js

@@ -2,6 +2,13 @@
 /*global $, document, $SCRIPT_ROOT */
 /* vim: set et sts=4: */
 
+$.valHooks.textarea = {
+    get: function (elem) {
+        "use strict";
+        return elem.value.replace(/\r?\n/g, "\r\n");
+    }
+};
+
 $(document).ready(function () {
     "use strict";
 
@@ -23,7 +30,7 @@ $(document).ready(function () {
         var project = $("#project").val(),
             urldata = $("input#workdir").val() + "/" + project;
         $("#status").empty();
-        $("#push").hide();
+	$("#commit").hide();
         $("#flash").empty();
         if (project === "") {
             $("#status").append("<h2>Please select one project...</h2><br/><br/>");
@@ -48,7 +55,7 @@ $(document).ready(function () {
                     //alert(message);
                     $("#status").append("<p>" + message + "</p>");
                     if (data.dirty) {
-                        $("#push").show();
+                        $("#commit").show();
                         $("#status").append("<br/><h2>Display Diff for current Project</h2>");
                         $("#status").append("<p style='font-size:15px;'>You have changes in your project." +
                             " <a href='#' id='viewdiff'"
@@ -162,9 +169,9 @@ $(document).ready(function () {
         checkout("0");
         return false;
     });
-    $("#commit").click(function () {
+    $("#commitbutton").click(function () {
         if ($("input#commitmsg").val() === "" ||
-                $("input#commitmsg").val() === "Enter message...") {
+                $("textarea#commitmsg").val() === "Enter message...") {
             $("#error").Popup("Please Enter the commit message", {type: 'alert', duration: 3000});
             return false;
         }
@@ -174,12 +181,12 @@ $(document).ready(function () {
         send = true;
         var project = $("#project").val();
         $("#imgwaitting").fadeIn('normal');
-        $("#commit").empty();
-        $("#commit").attr("value", "Wait...");
+        //$("#commit").empty();
+        $("#commitbbutton").attr("value", "Wait...");
         $.ajax({
             type: "POST",
-            url: $SCRIPT_ROOT + '/pushProjectFiles',
-            data: {project: $("input#workdir").val() + "/" + project, msg: $("input#commitmsg").val()},
+            url: $SCRIPT_ROOT + '/commitProjectFiles',
+            data: {project: $("input#workdir").val() + "/" + project, msg: $("textarea#commitmsg").val()},
             success: function (data) {
                 if (data.code === 1) {
                     if (data.result !== "") {
@@ -187,19 +194,46 @@ $(document).ready(function () {
                     } else {
                         $("#error").Popup("Commit done!", {type: 'confirm', duration: 3000});
                     }
+                    $("#commit").hide();
                     gitStatus();
                 } else {
                     $("#error").Popup(data.result, {type: 'error'});
                 }
                 $("#imgwaitting").hide();
-                $("#commit").empty();
-                $("#commit").attr("value", "Commit");
+                $("#commitmsg").empty();
+                $("#commitbutton").attr("value", "Commit");
                 send = false;
             }
         });
         return false;
     });
-
+    $("#push").click(function () {
+        if (send) {
+            return false;
+        }
+        send = true;
+        var project = $("#project").val();
+        $.ajax({
+            type: "POST",
+	    url: $SCRIPT_ROOT + '/pushProjectFiles',
+	    data: {project: $("input#workdir").val() + "/" + project},
+	    success: function (data) {
+	        if (data.code === 1) {
+	            if (data.result !== "") {
+                        $("#error").Popup(data.result, {type: 'error', duration: 5000});
+                    } else {
+                        $("#error").Popup("The local commits have correctly been saved on the server", {type: 'confirm', duration: 3000});
+                    }
+                    gitStatus();
+                } else {
+                    $("#error").Popup(data.result, {type: 'error'});
+                }
+		$("#push").hide();
+		send = false;
+	    }
+        });
+	return false;
+    });
     /*
     $("#pullbranch").click(function (){
       if (send){

slapos/runner/static/js/scripts/softwareFolder.js

@@ -15,7 +15,7 @@ $(document).ready(function () {
         softwareDisplay = true,
         projectDir = $("input#project").val(),
         workdir = $("input#workdir").val(),
-        currentProject = workdir + "/" + projectDir.replace(workdir, "").split('/')[1],
+        currentProject = "workspace/" + projectDir.replace(workdir, "").split('/')[1],
         send = false,
         edit = false,
         ajaxResult = false,
@@ -31,6 +31,7 @@ $(document).ready(function () {
     var MIN_TABITEM_WIDTH = 61; //The minimum size of tabItem
     var MAX_TAB_NUMBER = 10; //The maximum number of tab that could be opened
 
+
     function alertStatus (jqXHR) {
       if (jqXHR.status == 404) {
           $("#error").Popup("Requested page not found. [404]", {type: 'error'});
@@ -264,15 +265,26 @@ $(document).ready(function () {
           editorlist[activeToken].changed = true;
           $(activeSpan).html("*" + $(activeSpan).html());
         }
+        if (!beforeunload_warning_set) {
+          window.onbeforeunload = function() { return "You have unsaved changes"; };
+          beforeunload_warning_set = true;
+        }
       });
       editor.commands.addCommand({
-        name: 'myCommand',
+        name: 'SaveText',
         bindKey: {win: 'Ctrl-S',  mac: 'Command-S'},
         exec: function(editor) {
           $("#save").click();
         },
         readOnly: false // false if this command should not apply in readOnly mode
       });
+      editor.commands.addCommand({
+        name: 'Fullscreen',
+        bindKey: {win: 'Ctrl-E',  mac: 'Command-E'},
+        exec: function(editor) {
+            $("#fullscreen").click();
+        }
+      });
     }
 
     function getCurrentEditor () {
@@ -302,7 +314,7 @@ $(document).ready(function () {
     function switchContent() {
         if (!softwareDisplay) {
             $("span.swith_btn").empty();
-            $("span.swith_btn").append("Workspace");
+            $("span.swith_btn").append("Working dir");
             $('#fileTreeFull').show();
             $('#fileTree').hide();
         } else {
@@ -814,7 +826,7 @@ $(document).ready(function () {
     modelist = require("ace/ext/modelist");
     config = require("ace/config");
     initTree('#fileTree', currentProject, 'pfolder');
-    initTree('#fileTreeFull', 'workspace');
+    initTree('#fileTreeFull', 'runner_workdir');
     //bindContextMenu('#fileTree');
     $("#info").append("Current work tree: " + base_path());
 
@@ -827,6 +839,8 @@ $(document).ready(function () {
       if ($("#tabControl div.item").length === 0) {
         return false;
       }
+      beforeunload_warning_set = false;
+      window.onbeforeunload = function() { return; };
       var hash = getActiveToken();
       if (editorlist[hash].busy) {
           return false;
@@ -901,6 +915,17 @@ $(document).ready(function () {
       $("#option").click();
       return false;
     });
+    $("a#addflist").click(function(){
+      addToFavourite(current_file);
+      $("#option").click();
+      return false;
+    });
+
+    $("a#find").click(function(){
+      config.loadModule("ace/ext/searchbox", function(e) {e.Search(editor)});
+      $("#option").click();
+      return false;
+    });
 
     $("a#find").click(function () {
       if ($("#tabControl div.item").length === 0) {
@@ -924,4 +949,10 @@ $(document).ready(function () {
       return false;
     });
 
+    $("#fullscreen").click(function(){
+          $("body").toggleClass("fullScreen");
+          $("#editor").toggleClass("fullScreen-editor");
+          editor.resize();
+    });
+
 });

slapos/runner/static/js/scripts/viewlog.js

@@ -6,8 +6,19 @@
 $(document).ready(function () {
     "use strict";
 
+    function getQueryVariable(variable) {
+      var query = window.location.search.substring(1);
+      var vars = query.split("&");
+      for (var i=0;i<vars.length;i++) {
+        var pair = vars[i].split("=");
+        if (pair[0] == variable) {
+          return pair[1];
+        }
+      }
+    }
+
     // Current_log is not used for auto displaying mode, only for manual reload of log file!!!
-    var current_log = 'instance.log',
+    var current_log = (getQueryVariable("logfile") !== undefined)? getQueryVariable("logfile") : "instance.log",
         sending,
         state,
         selectedFile = "",

slapos/runner/templates/account.html

@@ -11,7 +11,10 @@
 <div id="tabContainer">
   <ul>
     <li><a href="#tab1" class="active">Your personal information</a></li>
-    <!--<li><a href="#tab2">Configurations</a></li>-->
+    {% if params %}
+    <li><a href="#tab2">Build & Run configuration</a></li>
+    <li><a href="#tab3">Add user</a></li>
+    {% endif %}
   </ul><!-- //Tab buttons -->
   <div class="tabDetails">
     <div id="tab1" class="tabContents">
@@ -33,7 +36,7 @@
           <input type='password' name='cpassword' id='cpassword' value=''/>
           <div class='clear'></div>
           <br/>
-          <label for="rcode">Password Recover code:</label>
+          <label for="rcode">Password Recovery code:</label>
           <input type='password' name='rcode' id='rcode' value=''/>
           <span class="information"><a href="#" id="information" rel="tooltip">help ?</a></span>
           <div class='clear'></div>
@@ -45,8 +48,52 @@
         <input type="hidden" name="hasAccount" id="hasAccount" value="{{name}}"/>
       </form>
     </div>
-    <!--<div id="tab2" class="tabContents">
-    </div>-->
+    {% if params %}
+    <div id="tab2" class="tabContents">
+      <form class="slapgrid">
+        <div class='form'>
+          <h2>Which one do you want to run ?</h2>
+          <label for="run_software">Build :</label>
+          <input type='checkbox' name='run_software' id='run_software' value='run_software' {% if params.run_software -%}checked{% endif %}>
+          <div class='clear'></div>
+          <label for="run_instance">Run :</label>
+          <input type='checkbox' name='run_instance' id='run_instance' value='run_instance' {% if params.run_instance -%}checked{% endif %}>
+          <div class='clear'></div>
+          <h2>How many tries ?</h2>
+          <label for="max_run_software">Max times for Build :</label>
+          <input type='text' name='max_run_software' id='max_run_software' value="{{params.max_run_software}}">
+          <div class='clear'></div>
+          <label for="max_run_instance">Max times for Run :</label>
+          <input type='text' name='max_run_instance' id='max_run_instance' value="{{params.max_run_instance}}">
+          <div class='clear'></div>
+          <br/>
+          <label></label>
+          <input type="submit" name="save" id ="save" value="Save config" class="button"/>
+          <div class='clear'></div>
+        </div>
+      </form>
+    </div>
+    <div id="tab3" class="tabContents">
+      <form class="slapgrid">
+        <div class='form'>
+          <label for="username">New user name :</label>
+          <input type='text' name='username' id='new_username'/>
+          <div class='clear'></div>
+          <label for="password">New password :</label>
+          <input type='password' name='password' id='new_password'/>
+          <div class='clear'></div>
+          <label for="new_rcode">Password Recovery code:</label>
+          <input type='password' name='new_rcode' id='new_rcode' value=''/>
+          <span class="information"><a href="#" id="information" rel="tooltip">help ?</a></span>
+          <div class='clear'></div>
+          <br/>
+          <label></label>
+          <input type="submit" name="add_user" id="add_user" value="Add new user" class="button"/>
+          <div class='clear'></div>
+        </div>
+      </form>
+    </div>
+    {% endif %}
 </div>
 </div>
 {% if username %}<div id="file_info" class="file_info">leave passwords blank to preserve your current password...

slapos/runner/templates/instanceInspect.html

@@ -20,6 +20,7 @@
       <li><a href="#tab2">Connection Information</a></li>
       <li><a href="#tab3" id="parameterTab">Parameters</a></li>
       <li><a href="#tab4" id="instancetabfiles">Partitions Content</a></li>
+      <li><a href="#tab5">Monitoring</a></li>
     </ul><!-- //Tab buttons -->
     <div class="tabDetails">
       <div id="tab1" class="tabContents">
@@ -132,6 +133,16 @@
         </h2>
       {%endif%}
         </div><!-- end tab4 -->
+      <div id="tab5" class="tabContents">
+        <h2>Monitoring interface</h2>
+        <div class="clear"></div>
+        <p>By clicking on the next link, you can access to the monitoring interface of the instance running inside the webrunner.</p>
+        <div class="clear"></div>
+        <p><b>Notice</b> that you have to extend the stack monitor in your software release (as explained <a href="http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/stack/monitor/README.txt?js=1">here</a>) for this weblink to work.
+        <div class="clear"></div>
+        <p><b><a href="{{ g.instance_monitoring_url }}" target="_blank">{{ g.instance_monitoring_url }}</a></b></p>
+        <div class="clear"></div>
+      </div><!-- end tab5 -->
     </div>
 </div>
 <!-- This contains the hidden content for inline calls -->

slapos/runner/templates/layout.html

@@ -36,6 +36,7 @@
     {% if request.path != '/login' %}
     <script src="{{ url_for('static', filename='js/ace/ace.js') }}" type="text/javascript" charset="utf-8"></script>
     <script src="{{ url_for('static', filename='js/ace/ext-modelist.js') }}" type="text/javascript" charset="utf-8"></script>
+    <script src="{{ url_for('static', filename='js/ace/ext-language_tools.js') }}" type="text/javascript" charset="utf-8"></script>
     <script src="{{ url_for('static', filename='js/scripts/process.js') }}" type="text/javascript" charset="utf-8"></script>
     <script type="text/javascript">
   		$(document).ready(function() {
@@ -66,8 +67,6 @@
         <div class="block_header">
           <a href="{{ url_for('home') }}" style="float:left;" id="home" title="Home"><img alt="" src="{{ url_for('static', filename='images/home.png') }}" /></a>
 	        <div class="line"></div>
-	        <a href="{{ url_for('dologout') }}" style="float:left" title="Close your session"><img alt="" src="{{ url_for('static', filename='images/logout.png') }}" /></a>
-	        <div class="line"></div>
           <h2 class="info">{% block title %}{% endblock %} - {{session.title}}</h2>
     	    <div class="run">
             <div id="running" style="display:none">
@@ -81,7 +80,7 @@
             <li><a href="{{ url_for('editCurrentProject') }}">Editor</a></li>
             <li><a href="{{ url_for('inspectInstance') }}">Services</a></li>
             <li><a href="{{ url_for('viewLog') }}">Logs</a></li>
-            <li><a href="{{ url_for('viewLog') }}">Terminal</a></li>
+            <li><a href="{{ url_for('shell') }}">Terminal</a></li>
             <li><a href="{{ url_for('manageRepository')}}#tab2">Git</a></li>
             <li class='right_menu main_menu'><a href="#"></a>
               <ul>
@@ -94,9 +93,7 @@
                 <li><a href="{{ url_for('runInstanceProfile') }}" id="instrun">Redeploy Services</a></li>
                 <li class='sep'></li>
                 <li><a href="{{ url_for('browseWorkspace') }}">Browse Workspace</a></li>
-                <li><a href="{{ url_for('inspectSoftware') }}">My Softwares Releases</a></li>
-                <li class='sep'></li>
-                <li><a href="{{ url_for('dologout') }}">Log out</a></li>
+                <li><a href="{{ url_for('inspectSoftware') }}">My Software Releases</a></li>
              </ul>
             </li>
             <li class='right_menu slapos_run' id="softrun"><a href="{{ url_for('runSoftwareProfile') }}"></a>

slapos/runner/templates/shell.html

+{% extends "layout.html" %}
+{% block body %}
+<iframe id="shellinabox" src="/shellinabox"></iframe>
+{% endblock %}

slapos/runner/templates/softwareFolder.html

@@ -29,7 +29,7 @@
 <!-- Definition of context menu -->
 <ul id="fileTreeMenu" class="contextMenu">
   <li class="edit"><a href="#edit">Edit</a></li>
-  <li class="view"><a href="#view">View this file</a></li>
+  <li class="view"><a href="#view">Open in popup</a></li>
   <li class="rename separator"><a href="#rename">Rename</a></li>
   <li class="delete "><a href="#delete">Delete</a></li>
   <li class="refresh separator"><a href="#refresh">Refresh</a></li>
@@ -76,7 +76,7 @@
   <ul class="inline">
     <li><a id='getmd5' href="#">Get or Update md5sum</a></li>
     <li><a id='addflist' href="#">Add to favourites</a></li>
-    <li><a id='addflist' href="#">Full screen</a></li>
+    <li><a id='fullscreen' title="Show Editor in Fullscreen. Hint: Use Ctrl+E" href="#">Full screen &nbsp;&nbsp;[Ctrl+E]</a></li>
     <li><a id='find' href="#">Find in file &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;[Ctrl+F]</a></li>
     <li><a id='replace' href="#">Replace in file &nbsp;&nbsp;[Ctrl+H]</a></li>
   </ul>

slapos/runner/templates/viewLog.html

@@ -37,7 +37,7 @@
           <p>Processing</p>
           <div class="clear"></div>
         </div>
-        <p>SlapOS rebuild your software from source, allowing you to easily patch or add any free software. <a href="#">Lean how!</a></p>
+        <p>SlapOS rebuild your software from source, allowing you to easily patch or add any free software. <a href="{{ url_for('viewLog', logfile='software.log') }}">Learn how!</a></p>
       </div>
 
       <h2 class="instance">Running State</h2>
@@ -49,7 +49,7 @@
           <p>Waiting for starting</p>
           <div class="clear"></div>
         </div>
-        <p>SlapOS configure your running environment to match your needs. <a href="#">Lean how!</a></p>
+        <p>SlapOS configure your running environment to match your needs. <a href="{{ url_for('viewLog', logfile='instance.log') }}">Learn how!</a></p>
       </div>
 
       <div class="separator"></div>

slapos/runner/utils.py

@@ -2,12 +2,15 @@
 # vim: set et sts=2:
 # pylint: disable-msg=W0311,C0301,C0103,C0111,W0141,W0142
 
+import ConfigParser
+import json
 import logging
 import md5
 import multiprocessing
+import os
 import re
 import shutil
-import os
+import thread
 import time
 import urllib
 from xml.dom import minidom
@@ -15,6 +18,8 @@ from xml.dom import minidom
 import xml_marshaller
 from flask import jsonify
 
+from slapos.runner.gittools import cloneRepo
+
 from slapos.runner.process import Popen, isRunning, killRunningProcess
 from slapos.htpasswd import HtpasswdFile
 import slapos.slap
@@ -23,6 +28,7 @@ import slapos.slap
 
 logger = logging.getLogger('werkzeug')
 
+TRUE_VALUES = (1, '1', True, 'true', 'True')
 
 html_escape_table = {
   "&": "&",
@@ -32,6 +38,19 @@ html_escape_table = {
   "<": "&lt;",
 }
 
+def getBuildAndRunParams(config):
+  json_file = os.path.join(config['etc_dir'], 'config.json')
+  json_params = json.load(open(json_file))
+  return json_params
+
+
+def saveBuildAndRunParams(config, params):
+  """XXX-Nico parameters have to be correct.
+  Works like that because this function do not care
+  about how you got the parameters"""
+  json_file = os.path.join(config['etc_dir'], 'config.json')
+  open(json_file, "w").write(json.dumps(params))
+
 
 def html_escape(text):
   """Produce entities within text."""
@@ -94,6 +113,25 @@ def saveSession(config, account):
     return str(e)
 
 
+def getRcode(config):
+  parser = ConfigParser.ConfigParser()
+  try:
+    parser.read(config['knowledge0_cfg'])
+    return parser.get('public', 'recovery-code')
+  except (ConfigParser.NoSectionError, IOError) as e:
+    return None
+
+
+def createNewUser(config, name, passwd):
+  htpasswdfile = os.path.join(config['etc_dir'], '.htpasswd')
+  if os.path.exists(htpasswdfile):
+    htpasswd = HtpasswdFile(htpasswdfile)
+    htpasswd.update(name, passwd)
+    htpasswd.save()
+    return True
+  return False
+
+
 def getCurrentSoftwareReleaseProfile(config):
   """
   Returns used Software Release profile as a string.
@@ -231,7 +269,7 @@ def isSoftwareRunning(config=None):
   return isRunning('slapgrid-sr')
 
 
-def runSoftwareWithLock(config):
+def runSoftwareWithLock(config, lock=True):
   """
     Use Slapgrid to compile current Software Release and wait until
     compilation is done
@@ -243,24 +281,21 @@ def runSoftwareWithLock(config):
   if not os.path.exists(config['software_root']):
     os.mkdir(config['software_root'])
   stopProxy(config)
-  removeProxyDb(config)
   startProxy(config)
   logfile = open(config['software_log'], 'w')
   if not updateProxy(config):
     return False
-  # Accelerate compilation by setting make -jX
-  # XXX-Marco can have issues with implicit dependencies or recursive makefiles. should be configurable.
-  environment = os.environ.copy()
-  environment['MAKEFLAGS'] = '-j%r' % multiprocessing.cpu_count()
   slapgrid = Popen([config['slapgrid_sr'], '-vc',
                     '--pidfile', slapgrid_pid,
                     config['configuration_file_path'], '--now', '--develop'],
-                   stdout=logfile, env=environment,
-                   name='slapgrid-sr')
-  slapgrid.wait()
-  #Saves the current compile software for re-use
-  config_SR_folder(config)
-  return True
+                   stdout=logfile, name='slapgrid-sr')
+  if lock:
+    slapgrid.wait()
+    #Saves the current compile software for re-use
+    config_SR_folder(config)
+    return ( True if slapgrid.returncode == 0 else False )
+  else:
+    return False
 
 
 def config_SR_folder(config):
@@ -276,12 +311,16 @@ def config_SR_folder(config):
       if len(cfg) != 2:
         continue  # there is a broken config file
       list.append(cfg[1])
-  folder_list = os.listdir(config['software_root'])
+  if os.path.exists(config['software_root']):
+    folder_list = os.listdir(config['software_root'])
+  else:
+    return
   if not folder_list:
     return
   current_project = open(os.path.join(config['etc_dir'], ".project")).read()
-  projects = current_project.split('/')
-  name = projects[-2]
+  if current_project[-1] == '/':
+     current_project = current_project[:-1]
+  name = current_project.split('/')[-1]
   for folder in folder_list:
     if folder in list:
       continue  # this folder is already registered
@@ -321,7 +360,7 @@ def isInstanceRunning(config=None):
   return isRunning('slapgrid-cp')
 
 
-def runInstanceWithLock(config):
+def runInstanceWithLock(config, lock=True):
   """
     Use Slapgrid to deploy current Software Release and wait until
     deployment is done.
@@ -337,9 +376,12 @@ def runInstanceWithLock(config):
   slapgrid = Popen([config['slapgrid_cp'], '-vc',
                     '--pidfile', slapgrid_pid,
                     config['configuration_file_path'], '--now'],
-                   stdout=logfile, name='slapgrid-cp')
-  slapgrid.wait()
-  return True
+                    stdout=logfile, name='slapgrid-cp')
+  if lock:
+    slapgrid.wait()
+    return ( True if slapgrid.returncode == 0 else False )
+  else:
+    return False
 
 
 def getProfilePath(projectDir, profile):
@@ -692,6 +734,7 @@ def realpath(config, path, check_exist=True):
     'software_root': config['software_root'],
     'instance_root': config['instance_root'],
     'workspace': config['workspace'],
+    'runner_workdir': config['runner_workdir'],
     'software_link': config['software_link']
   }
   if key not in allow_list:
@@ -731,3 +774,101 @@ def readParameters(path):
       return str(e)
   else:
     return "No such file or directory: %s" % path
+
+
+def isSoftwareReleaseReady(config):
+  """Return 1 if the Software Release has
+  correctly been deployed, 0 if not,
+  and 2 if it is currently deploying"""
+  project = os.path.join(config['etc_dir'], '.project')
+  if not os.path.exists(project):
+    return "0"
+  path = open(project, 'r').readline().strip()
+  software_name = path
+  if software_name[-1] == '/':
+    software_name = software_name[:-1]
+  software_name = software_name.split('/')[-1]
+  config_SR_folder(config)
+  if os.path.exists(os.path.join(config['runner_workdir'],
+      'softwareLink', software_name, '.completed')):
+    if config['autorun'] in TRUE_VALUES:
+      runSlapgridUntilSuccess(config, 'instance')
+    return "1"
+  else:
+    if isSoftwareRunning(config):
+      return "2"
+    elif config['auto_deploy'] in TRUE_VALUES:
+      configNewSR(config, path)
+      runSoftwareWithLock(config)
+      config_SR_folder(config)
+      time.sleep(15)
+      if config['autorun'] in TRUE_VALUES:
+        runSlapgridUntilSuccess(config, 'instance')
+      return "2"
+    else:
+      return "0"
+
+
+def cloneDefaultGit(config):
+  """Test if the default git has been downloaded yet
+  If not, download it in read-only mode"""
+  default_git = os.path.join(config['runner_workdir'],
+    'project', 'default_repo')
+  if not os.path.exists(default_git):
+    data = {'path': default_git,
+            'repo': config['default_repo'],
+    }
+    cloneRepo(data)
+
+
+def buildAndRun(config):
+  runSoftwareWithLock(config)
+  runInstanceWithLock(config)
+
+
+def runSlapgridUntilSuccess(config, step):
+  """Run slapgrid-sr or slapgrid-cp several times,
+  in the maximum of the constant MAX_RUN_~~~~"""
+  params = getBuildAndRunParams(config)
+  if step == "instance":
+    max_tries = (params['max_run_instance'] if params['run_instance'] else 0)
+    runSlapgridWithLock = runInstanceWithLock
+  elif step == "software":
+    max_tries = (params['max_run_software'] if params['run_software'] else 0)
+    runSlapgridWithLock = runSoftwareWithLock
+  else:
+    return -1
+  counter_file = os.path.join(config['runner_workdir'], '.turn-left')
+  open(counter_file, 'w+').write(str(max_tries))
+  counter = max_tries
+  slapgrid = True
+  # XXX-Nico runSoftwareWithLock can return 0 or False (0==False)
+  while counter > 0:
+    counter -= 1
+    slapgrid = runSlapgridWithLock(config)
+    if slapgrid:
+      break
+    times_left = int(open(counter_file).read()) - 1
+    if times_left > 0 :
+      open(counter_file, 'w+').write(str(times_left))
+      counter = times_left
+    else :
+      counter = 0
+  max_tries -= counter
+  # run instance only if we are deploying the software release,
+  # if it is defined so, and sr is correctly deployed
+  if step == "software" and params['run_instance'] and slapgrid:
+    return (max_tries, runSlapgridUntilSuccess(config, "instance"))
+  else:
+    return max_tries
+
+
+def setupDefaultSR(config):
+  """If a default_sr is in the parameters,
+  and no SR is deployed yet, setup it
+  also run SR and Instance if required"""
+  project = os.path.join(config['etc_dir'], '.project')
+  if not os.path.exists(project) and config['default_sr'] != '':
+    configNewSR(config, config['default_sr'])
+  if config['auto_deploy']:
+    thread.start_new_thread(buildAndRun, (config,))