fluentTest: initial commit

software/erp5testnode/testsuite/fluentTest/Caddyfile.in

+{{local_ip}}:4443
+
+  proxy / 0.0.0.0:9443
+

software/erp5testnode/testsuite/fluentTest/fluentd-conf.cnf.in

+<source>
+  @type forward
+</source>
+
+#<source>
+#  @type dummy
+#  tag dummy
+#  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+#</source>
+
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+#<source>
+#  @type http
+#  @id http_input
+#  port 8888
+#</source>
+
+
+## output tag=sensor.** to Wendelin
+<match **>
+  @type wendelin
+  @id wendelin_out
+
+  streamtool_uri http://{{local_ip}}:4443
+ 
+
+
+  # all parameters of BufferedOutput & Output classes are supported too, e.g.
+  # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
+  # their code near:
+  #
+  #   https://github.com/fluent/fluentd/blob/master/lib/fluent/output.rb#L182
+  #
+  # logging setup description:
+  #
+  #   http://docs.fluentd.org/articles/logging
+
+  buffer_type       memory
+  buffer_chunk_limit 1MB
+  #buffer_path       "#{ENV['HOME']/var}"
+  flush_interval    1s
+  num_threads 1
+</match>
+
+
+
+# ---- monitoring & debugging ----
+
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  @type monitor_agent
+  @id monitor_agent_input
+
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  @type debug_agent
+  @id debug_agent_input
+
+  bind 127.0.0.1
+  port 24230
+</source>
+
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  @type stdout
+  @id stdout_output
+</match>
+
+
+## match fluent's internal events
+#<match fluent.**>
+#  @type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  @type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+

software/erp5testnode/testsuite/fluentTest/instance-caddy-fluentd.cfg.in

+[buildout]
+parts =
+  directory
+  runTestSuite-instance
+  fluentdConfig
+  fluentd-service
+  caddy-service
+  publish-connection-information
+  runTest-instance
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+run = $${:var}/run
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promise = $${:etc}/promise/
+www = $${:srv}/www
+home = $${:etc}/home
+ssl = $${:etc}/ssl
+
+[runTestSuite-instance]
+recipe = slapos.recipe.template
+url = ${template-runTestSuite:output}
+output = $${directory:bin}/runTestSuite
+buildout-directory = $${buildout:directory}
+mode = 0700
+
+
+[runTest-instance]
+recipe = slapos.recipe.template
+url = ${template-test:output}
+output = $${directory:bin}/testIngestion.py
+buildout-directory = $${buildout:directory}
+mode = 0700
+
+
+#################################
+# fluentd service
+#################################
+
+
+[fluentdConfig]
+recipe = slapos.recipe.template:jinja2
+template = ${template-fluentdConfig:location}/${template-fluentdConfig:filename}
+rendered = $${directory:etc}/fluentd-conf.cnf
+mode = 0600
+context = 
+  key local_ip caddy-configuration:local_ip
+
+
+[fluentd-service]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/fluentd-service
+path = ${fluentd:location}
+command-line    =  ${fluentd:location}/bin/fluentd 
+    -c $${directory:etc}/fluentd-conf.cnf  -p ${fluentd-plugin-dev-repository:location}/lib/fluent/plugin/ 
+environment =
+  GEM_PATH= ${fluentd:location}/lib/ruby/gems/1.8/
+output = $${:wrapper-path}  
+
+#################################
+# caddy service
+#################################
+
+[caddy-service]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-service:output}
+rendered = $${directory:services}/caddy
+mode = 0700
+context =
+  key caddy_exec caddy-exec-dict:caddy-exec-file
+  key caddy_file caddy-configuration:rendered
+  key pidfile_dir directory:etc
+
+[caddy-exec-dict]
+caddy-exec-file = ${gowork:bin}/caddy
+  
+[caddy-configuration]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddyfile:output}
+rendered = $${directory:etc}/Caddyfile
+mode = 0600
+access_log = $${directory:log}/caddy-access.log
+error_log = $${directory:log}/caddy-error.log
+local_ip = $${slap-network-information:local-ipv4}
+context = 
+  key local_ip caddy-configuration:local_ip
+  
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+secure_access = http://$${caddy-configuration:local_ip}:4443}
+
+
+

software/erp5testnode/testsuite/fluentTest/instance.cfg.in

+#############################
+#
+# Deploy caddy instance
+#
+#############################
+[buildout]
+parts =
+  switch-softwaretype
+
+#  publish-connection-parameter
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:caddy}
+caddy = $${dynamic-template-caddy-fluentd:rendered}
+
+[dynamic-template-caddy-fluentd]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-fluentd:output}
+rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+filename = instance-caddy-fluentd.cfg
+
+[slap-connection]
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+[instance-parameter]
+# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
+# We use the slapconfiguration recipe with a few parameters (partition id,
+# computer id, certificate, etc).
+# It will then authenticate to SlapOS Master and fetch the instance parameters.
+# The parameters are accessible from {instance-parameter:configuration.name-of-parameter}
+# Always the same. Just copy/paste.
+# See docstring of slapos.cookbook:slapconfiguration for more information.
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/runTestSuite.in

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import argparse, os, re, shutil, subprocess, sys, traceback
+from erp5.util import taskdistribution
+
+import sys
+
+test_path = '$${runTest-instance:buildout-directory}/bin/'
+
+sys.path.append(test_path)
+import testIngestion
+
+def main():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title')
+  parser.add_argument('--test_node_title', help='The test node title')
+  parser.add_argument('--project_title', help='The project title')
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='ignored', type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites')
+  parser.add_argument('--frontend_url',
+                      help='The url of frontend of the test suite')
+  parser.add_argument('--target',
+                      help='Target OS to run tests on',
+                      type=str)
+  parser.add_argument('--target_version',
+                      help='Target OS version to use',
+                      type=str,)
+  parser.add_argument('--target_browser',
+                      help='The desired browser of the target OS to be used. Example: Firefox if target is Android.',
+                      type=str,)
+  parser.add_argument('--target_device',
+                      help='The desired device running the target OS. Example: iPad Simulator, if target is iOS.',
+                      type=str,)
+  parser.add_argument('--appium_server_auth',
+                      help='Combination of user and token to access SauceLabs service. (i.e. user:token)',
+                      type=str)
+
+  args = parser.parse_args()
+
+  
+  
+  
+  try:
+    test_suite_title = args.test_suite_title or args.test_suite
+    test_suite = args.test_suite
+    revision = args.revision
+
+    test_line_dict = {}
+    
+    ######!!!!!!!!!!!!!! uncomment this !!!!!!!!
+    # result.errors, result.failures, stream.read()
+    num_of_tests, errors, failures, stdout = testIngestion.main()
+    result = testIngestion.main()
+    
+    test_line_dict["testIngestion"] = {
+        'test_count': num_of_tests,
+        'error_count': len(errors),
+        'failure_count': len(failures),
+        'skip_count': 0,
+        'stdout': stdout,
+        'stderr': errors
+      }
+   
+   
+    tool = taskdistribution.TaskDistributor(portal_url=args.master_url)
+    test_result = tool.createTestResult(revision = revision,
+                                        test_name_list = test_line_dict.keys(),
+                                        node_title = args.test_node_title,
+                                        test_title = test_suite_title,
+                                        project_title = args.project_title)
+                                        
+  
+    
+    if test_result is None or not hasattr(args, 'master_url'):
+      print("test_result is NONE")
+      return
+    # report test results
+    while 1:
+      test_result_line = test_result.start()
+      if not test_result_line:
+        print 'No test result anymore.'
+        break
+
+      print 'Submitting: "%s"' % test_result_line.name
+      # report status back to Nexedi ERP5
+      test_result_line.stop(**test_line_dict[test_result_line.name])
+
+  except:
+    # Catch any exception here, to warn user instead of being silent,
+    # by generating fake error result
+    print traceback.format_exc()
+    result = dict(status_code=-1,
+                  command=url,
+                  stderr=traceback.format_exc(),
+                  stdout='')
+    # XXX: inform test node master of error
+    raise EnvironmentError(result)
+  
+  
+
+if __name__ == "__main__":
+    main()

software/erp5testnode/testsuite/fluentTest/software.cfg

+[buildout]
+
+extends =
+  ../../../../stack/slapos.cfg
+  ../../../../software/caddy/software.cfg
+  ../../../../software/fluentd/software.cfg
+
+index = https://pypi.python.org/simple/
+
+parts =
+  slapos-cookbook
+  eggs
+  caddy
+  fluentd
+  fluentd-plugin-dev-repository
+  instance-profile
+  template-runTestSuite
+  template-fluentdConfig
+  
+
+[instance-profile]
+recipe = slapos.recipe.template
+md5sum = 45c4adaae557bed5c25c24f4c4506b3d
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+mode = 0644
+
+[template-runTestSuite]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/runTestSuite.in
+#md5sum = ff66d13f73982e8257eb5535cdb541c7
+output = ${buildout:directory}/runTestSuite.in
+mode = 0644
+
+[template-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/testIngestion.py
+#md5sum = ff66d13f73982e8257eb5535cdb541c7
+output = ${buildout:directory}/testIngestion.py
+mode = 0644
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  erp5.util
+  ${lxml-python:egg}
+  requests
+interpreter = pythonwitheggs
+
+[template-fluentdConfig]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/fluentd-conf.cnf.in
+#md5sum = 3132b99d65c75b72fc6dacb702ddca0b
+filename = fluentd-conf.cnf.in
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
+[fluentd-plugin-dev-repository]
+recipe  = slapos.recipe.build:gitclone
+repository = https://lab.nexedi.com/Sokhoyan/fluent-plugin-wendelin.git
+branch = add-keep-alive
+# dir is pretty name as top-level -dev recipe
+git-executable = ${git:location}/bin/git
+
+[fluentd]
+gems =
+  fluentd
+  fluent-plugin-td
+  bundler
+  fluent-plugin-bin
+
+[template-caddy-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-caddy-service.sh.in
+md5sum = 3e09969c479931ca4df6546abbd1a117
+output = ${buildout:directory}/template-caddy-service.sh.in
+mode = 0644
+
+[template-caddyfile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/Caddyfile.in
+md5sum = 3132b99d65c75b72fc6dacb702ddca0b
+output = ${buildout:directory}/Caddyfile.in
+mode = 0644
+
+[template-caddy-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-caddy-fluentd.cfg.in
+md5sum = 599d4650e9b96ce15fceb512316651ae
+output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
+mode = 0644
+
+[caddy]
+recipe  = slapos.recipe.cmmi
+path    = ${go_github.com_mholt_caddy:location}
+go      = ${gowork:golang}/bin/go
+configure-command = :
+make-targets =
+make-binary = cd ${:path}/caddy  && ${:go} install -v
+environment =
+  PATH=${pkgconfig:location}/bin:${gowork:golang}/bin:${buildout:bin-directory}:%(PATH)s
+  GOPATH=${gowork:directory}
+output =  ${gowork:bin}/caddy
+
+[versions]
+slapos.recipe.template = 4.3
+erp5.util = 0.4.50

software/erp5testnode/testsuite/fluentTest/template-caddy-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+{{ caddy_exec }} -conf {{ caddy_file }} -pidfile {{pidfile_dir}}/caddy_pidfile
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/templates/empty.in

+{{ content }}
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/templates/index.html

+<h1>Welcome</h1>
\ No newline at end of file

software/erp5testnode/testsuite/fluentTest/testIngestion.py

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import unittest
+import requests
+from StringIO import StringIO
+
+import SimpleHTTPServer
+import SocketServer
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+import os
+
+import threading
+import time
+
+
+test_msg = "dummyInputSimpleIngest"
+url = "http://$${caddy-configuration:local_ip}:4443"
+
+caddy_pidfile = "$${directory:etc}/caddy_pidfile"
+
+posted_data = None
+all_data = []
+request_tag = ""
+with open(caddy_pidfile) as f:
+  caddy_pid = f.readline()
+
+class TestServerHandler(BaseHTTPRequestHandler):
+
+    def _set_headers(self):
+        self.send_response(200)
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+    def do_GET(self):
+        self._set_headers()
+        self.wfile.write("<html><body><h1>hi!</h1></body></html>")
+
+
+    def do_HEAD(self):
+        self._set_headers()
+
+    def do_POST(self):
+        content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+        post_data = self.rfile.read(content_length) # <--- Gets the data itself
+
+        self._set_headers()
+        self.wfile.write(post_data)
+
+        global posted_data
+        posted_data = post_data
+        print("post data from do_POST")
+        print(posted_data)
+        global all_data
+        all_data.append(post_data.split(" ")[1])
+        
+        global request_tag
+        request_tag = find_tag(self.requestline,"=", " ")
+
+class TestPost(unittest.TestCase):
+
+    posted_data = ""
+
+    def test_1_get(self):
+      print("############## TEST 1 ##############")
+      resp = requests.get(url)
+      self.assertEqual(resp.status_code, 200)
+      print (resp.status_code)
+
+    def test_2_ingest(self):
+      print("############## TEST 2 ##############")
+      start_fluentd_cat(test_msg, "tag_test_2")
+      time.sleep(10)
+      print("posted data from test 2")
+      print(posted_data)
+      if posted_data:
+        self.assertEqual(test_msg, posted_data.split(" ")[1])
+      else:
+        self.assertEqual(test_msg, posted_data)
+      time.sleep(10)
+      self.assertEqual(test_msg, posted_data.split(" ")[1])
+  
+    def test_3_keepAlive_on(self):
+      print("############## TEST 3 ##############")
+      s = requests.session()
+      print("check connection type ")
+      print(s.headers['Connection'])
+      self.assertEqual('keep-alive', s.headers['Connection'])
+
+    def test_4_delay_15_mins(self):
+      print("############## TEST 4 ##############")
+      time.sleep(900)
+      start_fluentd_cat("dummyInputDelay", "tag_test_4")
+      time.sleep(15)
+      self.assertEqual("dummyInputDelay", posted_data.split(" ")[1])
+
+    def test_5_caddy_restart(self):
+      print("############## TEST 5 ##############")
+
+      with open(caddy_pidfile) as f:
+        caddy_pid = f.readline()
+
+      time.sleep(10)
+      start_fluentd_cat("dummyInputCaddyRestart1", "tag_test_5_1")
+      time.sleep(10)
+
+      kill_caddy(caddy_pid)
+      time.sleep(10)
+
+      start_fluentd_cat("dummyInputCaddyRestart2 ", "tag_test_5_2")
+      time.sleep(10)
+      start_fluentd_cat("dummyInputCaddyRestart3 ", "tag_test_5_3")
+      time.sleep(10)
+      start_fluentd_cat("dummyInputCaddyRestart4 ", "tag_test_5_4")
+      time.sleep(130)
+
+      start_caddy(caddy_pid)
+      time.sleep(15)
+
+      self.assertTrue("dummyInputCaddyRestart1" in all_data)
+      self.assertTrue("dummyInputCaddyRestart2" in all_data)
+      self.assertTrue("dummyInputCaddyRestart3" in all_data)
+      self.assertTrue("dummyInputCaddyRestart4" in all_data)
+      
+    def test_6_check_diff_tags(self):
+      print("############## TEST 6 ##############")
+      
+      start_fluentd_cat("dummyInputTags_6_1", "test_Tag_6_1")
+      time.sleep(2)
+      self.assertEqual("test_Tag_6_1", request_tag)
+      
+      start_fluentd_cat("dummyInputTags_6_2", "test_Tag_6_2")
+      time.sleep(2)
+      self.assertEqual("test_Tag_6_2", request_tag)
+      
+      start_fluentd_cat("dummyInputTags_6_3", "test_Tag_6_3")
+      time.sleep(2)
+      self.assertEqual("test_Tag_6_3", request_tag)
+      
+def start_fluentd_cat(test_msg, tag):
+
+    os.environ["GEM_PATH"] ="$${fluentd-service:path}/lib/ruby/gems/1.8/"
+    
+    fluentd_cat_exec_comand = '$${fluentd-service:path}/bin/fluent-cat --none ' + tag
+    os.system("echo + " + test_msg + " | " + fluentd_cat_exec_comand)
+
+def kill_caddy(caddy_pid):
+    
+    print("caddy pid = ")
+    print(caddy_pid)
+    
+    kill_caddy_cmd = "kill -TSTP " + caddy_pid
+    os.system(kill_caddy_cmd)
+    print("Caddy is killed")
+
+def start_caddy(caddy_pid):
+    
+    start_caddy_cmd = "kill -CONT " + caddy_pid
+    os.system(start_caddy_cmd)
+    print("Caddy is restarted")
+
+def find_tag(s, start, end):
+  return (s.split(start))[1].split(end)[0]
+
+def main():
+  
+    port=9443
+    server_address = ('0.0.0.0', port)
+    httpd = HTTPServer(server_address, TestServerHandler)
+    thread = threading.Thread(target=httpd.serve_forever)
+    thread.start()
+    print 'Starting http...'
+  
+    time.sleep(15)
+
+    stream = StringIO()
+    runner = unittest.TextTestRunner(verbosity=2, stream=stream)
+    result = runner.run(unittest.makeSuite(TestPost))
+ 
+    
+    print 'Tests run ', result.testsRun
+    print 'Errors ', result.errors
+    print "Failures ", result.failures
+    stream.seek(0)
+    print 'Test output\n', stream.read() 
+    
+    time.sleep(30)
+    httpd.shutdown()
+    print(posted_data)
+    
+    print("all posted data : ")
+    print(all_data)
+
+    return result.testsRun, result.errors, result.failures, stream.getvalue()
+
+if __name__ == "__main__":
+  
+    main()
\ No newline at end of file