fluentTest: initial commit

software/fluentTest/Caddyfile.in

+{{local_ip}}:4443
+
+  proxy / 0.0.0.0:9443
+

software/fluentTest/fluentd-conf.cnf.in

+<source>
+  @type dummy
+  tag dummy
+  dummy {"message":"yaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaay"}
+</source>
+
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  @type http
+  @id http_input
+  port 8888
+</source>
+
+
+## output tag=sensor.** to Wendelin
+<match dummy.**>
+  @type wendelin
+  @id wendelin_out
+
+  streamtool_uri    http://10.0.46.242:4443
+  # TODO ^^^ do not check peer's certificate
+  user      zope
+  password  dbguylpn
+
+
+  # all parameters of BufferedOutput & Output classes are supported too, e.g.
+  # `buffer_type`, `flush_interval`, `num_threads`, `log_level`, etc - see
+  # their code near:
+  #
+  #   https://github.com/fluent/fluentd/blob/master/lib/fluent/output.rb#L182
+  #
+  # logging setup description:
+  #
+  #   http://docs.fluentd.org/articles/logging
+
+  buffer_type       memory
+  buffer_chunk_limit 1MB
+  #buffer_path       "#{ENV['HOME']/var}"
+  flush_interval    1s
+</match>
+
+
+
+# ---- monitoring & debugging ----
+
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  @type monitor_agent
+  @id monitor_agent_input
+
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  @type debug_agent
+  @id debug_agent_input
+
+  bind 127.0.0.1
+  port 24230
+</source>
+
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  @type stdout
+  @id stdout_output
+</match>
+
+
+## match fluent's internal events
+#<match fluent.**>
+#  @type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  @type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+

software/fluentTest/instance-caddy-fluentd.cfg.in

+[buildout]
+parts =
+  directory
+  runTestSuite-instance
+  fluentd-service
+  caddy-service
+  publish-connection-information
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+bin = $${buildout:directory}/bin
+srv = $${buildout:directory}/srv
+var = $${buildout:directory}/var
+run = $${:var}/run
+log = $${:var}/log
+scripts = $${:etc}/run
+services = $${:etc}/service
+promise = $${:etc}/promise/
+www = $${:srv}/www
+home = $${:etc}/home
+ssl = $${:etc}/ssl
+
+[runTestSuite-instance]
+recipe = slapos.recipe.template
+url = ${template-runTestSuite:output}
+output = $${directory:bin}/runTestSuite
+buildout-directory = $${buildout:directory}
+mode = 0700
+
+
+#################################
+# fluentd service
+#################################
+
+[fluentd-service]
+recipe  = slapos.cookbook:wrapper
+wrapper-path = $${directory:services}/fluentd-service
+command-line    =  ${fluentd:location}/bin/fluentd
+    -v
+    -c ${template-fluentdConfig:output}
+environment =
+  GEM_PATH= ${fluentd:location}/lib/ruby/gems/1.8/
+output = $${:wrapper-path}  
+
+#################################
+# caddy service
+#################################
+
+[caddy-service]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-service:output}
+rendered = $${directory:services}/caddy
+mode = 0700
+context =
+  key caddy_exec caddy-exec-dict:caddy-exec-file
+  key caddy_file caddy-configuration:rendered
+
+[caddy-exec-dict]
+caddy-exec-file = ${gowork:bin}/caddy
+  
+[caddy-configuration]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddyfile:location}/${template-caddyfile:filename}
+rendered = $${directory:etc}/Caddyfile
+mode = 0600
+access_log = $${directory:log}/caddy-access.log
+error_log = $${directory:log}/caddy-error.log
+local_ip = $${slap-network-information:local-ipv4}
+context = 
+  key local_ip caddy-configuration:local_ip
+
+
+[publish-connection-information]
+recipe = slapos.cookbook:publish
+secure_access = http://$${caddy-configuration:local_ip}:4443}
+
+
+

software/fluentTest/instance.cfg.in

+#############################
+#
+# Deploy caddy instance
+#
+#############################
+[buildout]
+parts =
+  switch-softwaretype
+
+#  publish-connection-parameter
+
+# Define egg directories to be the one from Software Release
+# (/opt/slapgrid/...)
+# Always the same.
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[switch-softwaretype]
+recipe = slapos.cookbook:softwaretype
+default = $${:caddy}
+caddy = $${dynamic-template-caddy-fluentd:rendered}
+
+[dynamic-template-caddy-fluentd]
+recipe = slapos.recipe.template:jinja2
+template = ${template-caddy-fluentd:output}
+rendered = $${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
+filename = instance-caddy-fluentd.cfg
+
+[slap-connection]
+computer-id = $${slap_connection:computer_id}
+partition-id = $${slap_connection:partition_id}
+server-url = $${slap_connection:server_url}
+software-release-url = $${slap_connection:software_release_url}
+key-file = $${slap_connection:key_file}
+cert-file = $${slap_connection:cert_file}
+
+[instance-parameter]
+# Fetch arbitrary parameters defined by the user in SlapOS Master for his instance.
+# We use the slapconfiguration recipe with a few parameters (partition id,
+# computer id, certificate, etc).
+# It will then authenticate to SlapOS Master and fetch the instance parameters.
+# The parameters are accessible from {instance-parameter:configuration.name-of-parameter}
+# Always the same. Just copy/paste.
+# See docstring of slapos.cookbook:slapconfiguration for more information.
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap_connection:computer_id}
+partition = $${slap_connection:partition_id}
+url = $${slap_connection:server_url}
+key = $${slap_connection:key_file}
+cert = $${slap_connection:cert_file}
\ No newline at end of file

software/fluentTest/runTestSuite.in

+#!${buildout:directory}/bin/${eggs:interpreter}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+import argparse, os, re, shutil, subprocess, sys, traceback
+import SimpleHTTPServer
+import SocketServer
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+from erp5.util import taskdistribution
+from erp5.util.testsuite import TestSuite as BaseTestSuite
+
+
+
+
+#class Server(BaseHTTPRequestHandler):
+#    def _set_headers(self):
+#        self.send_response(200)
+#        self.send_header('Content-type', 'text/html')
+#        self.end_headers()
+
+#    def do_GET(self):
+#        self._set_headers()
+#        self.wfile.write("<html><body><h1>hi!</h1></body></html>")
+
+#    def do_HEAD(self):
+#        self._set_headers()
+        
+#    def do_POST(self):
+       
+ #       content_length = int(self.headers['Content-Length']) # <--- Gets the size of data
+#        post_data = self.rfile.read(content_length) # <--- Gets the data itself
+       # print(post_data)
+#        self._set_headers()
+#        self.wfile.write(post_data)
+        
+
+#def run(server_class=HTTPServer, handler_class=Server, port=9443):
+#    
+#    test_list = ["testIntestion.py"]
+    
+    
+#    server_address = ('', port)
+#    httpd = server_class(server_address, handler_class)
+#    print 'Starting http...'
+#    httpd.serve_forever()
+
+
+class TestSuite(BaseTestSuite):
+
+  def run(self, test):
+    return self.runUnitTest(test)
+
+  def runUnitTest(self, *args, **kw):
+    try:
+      runUnitTest = os.environ.get('RUN_UNIT_TEST',
+                                   'runUnitTest')
+      args = tuple(shlex.split(runUnitTest)) + args
+      status_dict = self.spawn(*args, **kw)
+    except SubprocessError, e:
+      status_dict = e.status_dict
+    test_log = status_dict['stderr']
+    search = self.RUN_RE.search(test_log)
+    if search:
+      groupdict = search.groupdict()
+      status_dict.update(duration=float(groupdict['seconds']),
+                         test_count=int(groupdict['all_tests']))
+    search = self.STATUS_RE.search(test_log)
+    if search:
+      groupdict = search.groupdict()
+      status_dict.update(
+        error_count=int(groupdict['errors'] or 0),
+        failure_count=int(groupdict['failures'] or 0)
+                     +int(groupdict['unexpected_successes'] or 0),
+        skip_count=int(groupdict['skips'] or 0)
+                  +int(groupdict['expected_failures'] or 0))
+    return status_dict
+
+
+def run():
+  parser = argparse.ArgumentParser(description='Run a test suite.')
+  parser.add_argument('--test_suite', help='The test suite name')
+  parser.add_argument('--test_suite_title', help='The test suite title',
+                      default=None)
+  parser.add_argument('--test_node_title', help='The test node title',
+                      default=None)
+  parser.add_argument('--project_title', help='The project title',
+                      default=None)
+  parser.add_argument('--revision', help='The revision to test',
+                      default='dummy_revision')
+  parser.add_argument('--node_quantity', help='Number of parallel tests to run',
+                      default=1, type=int)
+  parser.add_argument('--master_url',
+                      help='The Url of Master controling many suites',
+                      default=None)
+
+  testList=["testIngestion.py"]
+
+  args = parser.parse_args()
+  master = taskdistribution.TaskDistributionTool(args.master_url)
+  test_suite_title = args.test_suite_title or args.test_suite
+  revision = args.revision
+  suite = TestSuite(1, test_suite=args.test_suite,
+                    node_quantity=args.node_quantity,
+                    revision=revision)
+
+  test_result = master.createTestResult(revision, testList,
+    args.test_node_title, suite.allow_restart, test_suite_title,
+    args.project_title)
+  if test_result is not None:
+    assert revision == test_result.revision, (revision, test_result.revision)
+    while suite.acquire():
+      test = test_result.start(suite.running.keys())
+      if test is not None:
+        suite.start(test.name, lambda status_dict, __test=test:
+          __test.stop(**status_dict))
+      elif not suite.running:
+        break
+

software/fluentTest/software.cfg

+[buildout]
+
+extends =
+  ../../stack/slapos.cfg
+  ../../software/caddy/software.cfg
+  ../../software/fluentd/software.cfg
+
+parts =
+  slapos-cookbook
+  eggs
+  fluentd
+  instance-profile
+  template-runTestSuite
+  template-fluentdConfig
+
+[instance-profile]
+recipe = slapos.recipe.template
+md5sum = 45c4adaae557bed5c25c24f4c4506b3d
+url = ${:_profile_base_location_}/instance.cfg.in
+output = ${buildout:directory}/instance.cfg
+mode = 0644
+
+[template-runTestSuite]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/runTestSuite.in
+#md5sum = ff66d13f73982e8257eb5535cdb541c7
+output = ${buildout:directory}/runTestSuite.in
+mode = 0644
+
+[eggs]
+recipe = zc.recipe.egg
+eggs =
+  erp5.util
+  selenium
+  ${lxml-python:egg}
+interpreter = pythonwitheggs
+
+[template-fluentdConfig]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/fluentd-conf.cnf.in
+#md5sum = ff66d13f73982e8257eb5535cdb541c7
+output = ${buildout:directory}/fluentd-conf.cnf.in
+mode = 0644
+
+[fluentd]
+gems +=
+  fluent-plugin-wendelin==0.3
+  fluent-plugin-bin
+
+[template-caddy-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/template-caddy-service.sh.in
+md5sum = 0119e5b6126215204c04285c2389fe1f
+output = ${buildout:directory}/template-caddy-service.sh.in
+mode = 0644
+
+[template-caddyfile]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/Caddyfile.in
+md5sum = 3132b99d65c75b72fc6dacb702ddca0b
+filename = Caddyfile.in
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
+[template-caddy-fluentd]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/instance-caddy-fluentd.cfg.in
+md5sum = 33652e8cdbf244911fc861c901a10af4
+output = ${buildout:directory}/instance-caddy-fluentd.cfg.in
+mode = 0644

software/fluentTest/template-caddy-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapgrid
+# BEWARE: It will be overwritten automatically
+
+{{ caddy_exec }} -conf {{ caddy_file }}
\ No newline at end of file

software/fluentTest/templates/empty.in

+{{ content }}
\ No newline at end of file

software/fluentTest/templates/index.html

+<h1>Welcome</h1>
\ No newline at end of file

software/fluentTest/testIngestion.py

+import unittest
+import requests
+
+
+class TestPost(unittest.TestCase):
+  
+    
+    def test_get(self):
+        resp = requests.get('http://10.0.46.242:4443')
+        self.assertEqual(resp.status_code, 200)
+        print (resp.status_code)
+    
+    
+    def test_post(self):
+        var_name_request = 'var1'
+        value_request = "dummydummydummy123"
+        req = requests.post('http://10.0.46.242:4443',data={var_name_request: value_request})
+        var_name_response = req.text.split('=')[0]
+        value_response = req.text.split('=')[1]
+        self.assertEqual(var_name_request, var_name_response)
+        self.assertEqual(value_request, value_response)
+        
+
+if __name__ == "__main__":
+    #unittest.main()
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestPost)
+    unittest.TextTestRunner(verbosity=2).run(suite)
\ No newline at end of file