stack/erp5: Add promises to check mariadb and apachedex reports

stack/erp5/buildout.cfg

@@ -52,6 +52,7 @@ extends =
   ../../component/coreutils/buildout.cfg
   ../../component/grep/buildout.cfg
   ../../component/dash/buildout.cfg
+  ../../component/bash/buildout.cfg
   ../../component/wget/buildout.cfg
   ../../component/aspell/buildout.cfg
   ../../component/cloudooo/buildout.cfg
@@ -100,6 +101,7 @@ parts +=
   percona-toolkit
   zabbix-agent
   dash
+  bash
   wget
   userhosts
   postfix
@@ -168,6 +170,10 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
 <= download-base
 mode = 755
 
+[mariadb-slowquery-check-script]
+<= download-base
+mode = 755
+
 [mariadb-slow-query-report-script]
 <= download-base
 mode = 755
@@ -232,6 +238,8 @@ context =
     key mariadb_link_binary template-mariadb:link-binary
     key zope_link_binary template-zope:link-binary
     key apache_location apache:location
+    key apdex_result_check_script apdex-result-check-script:target
+    key mariadb_slowquery_check_script mariadb-slowquery-check-script:target 
     key aspell_location aspell:location
     key bin_directory buildout:bin-directory
     key buildout_bin_directory buildout:bin-directory
@@ -242,6 +250,7 @@ context =
     key curl_location curl:location
     key cyrus_sasl_location cyrus-sasl:location
     key dash_location dash:location
+    key bash_location bash:location
     key dbus_glib_location dbus-glib:location
     key dbus_location dbus:location
     key dcron_location dcron:location
@@ -351,6 +360,9 @@ link-binary =
 [template-balancer]
 <= download-base
 
+[apdex-result-check-script]
+<= download-base
+
 [template-haproxy-cfg]
 <= download-base
 

stack/erp5/buildout.hash.cfg

@@ -19,11 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = dc974bd74cf967ae6250d81322629c44
+md5sum = 48fb6a0ed34cfc6de6b40acd7088c51c
+
+[mariadb-slowquery-check-script]
+filename = instance-mariadb-check-slowquery-result.sh.in 
+md5sun = c9cd394f3d031cef77da50610544d2b0
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 2126e64bc70893500dc24c29770e84c8
+md5sum = 9c34cba998ce0ed3b96e2604d2ed1cb5
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -75,7 +79,7 @@ md5sum = 0969fbb25b05c02ef3c2d437b2f4e1a0
 
 [template]
 filename = instance.cfg.in
-md5sum = e364ea67bfe786b6b6ebd6c4f0cd628a
+md5sum = cbe1df5dbd9b79ed7adc027fd183e186
 
 [monitor-template-dummy]
 filename = dummy.cfg
@@ -95,7 +99,11 @@ md5sum = a2377d5c53fd2a441ea713b428e4844b
 
 [template-balancer]
 filename = instance-balancer.cfg.in
-md5sum = 25d0ebecda78d1440dacd3495d53c928
+md5sum = 79fc39f7fbf13b1788adb5c33150dd80
+
+[apdex-result-check-script]
+filename = instance-balancer-check-apachedex-result.sh.in 
+md5sum = e1dd2f2884ea0e9f6f230419e032d927
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer-check-apachedex-result.sh.in

+#!{{ bash }}
+
+set -e
+
+apachedex_file='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
+apachedex_status_file={{ apdex_status_file }} 
+desired_threshold={{ default_threshold }}
+user_threshold=$(<{{ user_threshold }})
+
+if [ ! -z "$user_threshold" ]
+then
+  desired_threshold=$user_threshold
+fi
+
+# Check if the file is there
+if [ ! -s "$apachedex_file" ]; then 
+  # If file doesn't exists create one
+  # If it is empty check for modification time
+  if [ ! -f "$apachedex_file" ]; then
+    touch $apachedex_file
+  else
+    modified_date=`stat -c '%Z' $apachedex_file`
+    current_date=`date +%s`
+    if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
+    then 
+      echo "File modification date is greater than 30 hours"
+      json_content=`cat $apachedex_status_file`
+      message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
+      echo $message 
+      exit 2
+    else
+      echo "File is empty for now"
+    fi
+  fi
+else
+  # Check if the result exists
+  {
+    regex="apdex threshold<\/th><td>(.*)s<\/td>"
+    file_content=`cat $apachedex_file`
+    if [[ $file_content =~ $regex ]]
+    then
+      threshold=${BASH_REMATCH[1]}
+      threshold=${threshold:-0}
+      if [[ `echo "$threshold > $desired_threshold" | bc` -eq 1 ]]
+      then
+        echo "Your score is $threshold, Thanks for keeping it all clean"
+      else
+        echo "Threshold is lower than exptected: Expected was $desired_threshold and we current is $threshold"
+        exit 2
+      fi
+    else
+      echo "No threshold found in the result"
+    fi
+  } || {
+    echo "Cannot parse the apdex result"
+  }
+fi

stack/erp5/instance-balancer.cfg.in

@@ -287,7 +287,7 @@ apachedex = ${monitor-directory:private}/apachedex
 [{{ section('monitor-generate-apachedex-report') }}]
 recipe = slapos.cookbook:wrapper
 wrapper-path = ${monitor-directory:reports}/${:command}
-command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
+command-line = "{{ parameter_dict['run-apachedex-location'] }}" "{{ parameter_dict['apachedex-location'] }}" "${directory:apachedex}" ${monitor-publish-parameters:monitor-base-url}/private/apachedex --default "${apachedex-parameters:default}" --apache-log-list "${apachedex-parameters:apache-log-list}" --base-list "${apachedex-parameters:base-list}" --skip-base-list "${apachedex-parameters:skip-base-list}" --erp5-base-list "${apachedex-parameters:erp5-base-list}"
 command = apachedex_every_3_hour
 
 [apachedex-parameters]
@@ -299,6 +299,19 @@ default = ${monitor-directory:etc}/apdex_default
 base-list = ${monitor-directory:etc}/apdex_base_list
 skip-base-list = ${monitor-directory:etc}/apdex_skip_base_list
 erp5-base-list = ${monitor-directory:etc}/apdex_erp5_base_list
+apdex-promise-threshold = ${monitor-directory:etc}/apdex-promise-threshold
+
+[{{ section('monitor-promise-apachedex-result') }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['apdex-result-check-script'] }}
+rendered = ${monitor-directory:promises}/check-apachedex-result
+status-file = ${monitor-directory:private}/apachedex.report.json 
+context = 
+  raw default_threshold 0.7
+  raw bash {{ parameter_dict['bash'] }}/bin/bash
+  key apdex_file directory:apachedex
+  key apdex_status_file :status-file
+  key user_threshold apachedex-parameters:apdex-promise-threshold
 
 [monitor-instance-parameter]
 monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
@@ -310,6 +323,7 @@ instance-configuration =
   file apachedex-base-list ${apachedex-parameters:base-list}
   file apachedex-skip-base-list ${apachedex-parameters:skip-base-list}
   file apachedex-erp5-base-list ${apachedex-parameters:erp5-base-list}
+  file apachedex-erp5-promise-threshold ${apachedex-parameters:apdex-promise-threshold}
 
 [buildout]
 extends =

stack/erp5/instance-mariadb-check-slowquery-result.sh.in

+#!{{ bash }}
+set -e
+
+digest_file='{{ slow_query_digest }}/slowquery_digest.txt'
+slow_query_status_file='{{ slow_query_status }}'
+desired_max_query_threshold={{ default_threshold }}
+desired_slow_query_threshold=0
+
+max_queries_threshold='{{ max_queries_threshold  }}'
+if [ ! -z "$max_queries_threshold" ]
+then
+  desired_max_query_threshold=$max_queries_threshold
+fi
+
+slowest_queries_threshold='{{ slowest_queries_threshold  }}'
+if [ ! -z "$slowest_queries_threshold" ]
+then
+  desired_slow_query_threshold=$slowest_queries_threshold
+fi
+
+# Check if the file is there
+if [ ! -s "$digest_file" ]; then 
+  # If file doesn't exists create one
+  # If it is empty check for modification time
+  if [ ! -f "$digest_file" ]; then
+    touch $digest_file
+  else
+    modified_date=`stat -c '%Z' $digest_file`
+    current_date=`date +%s`
+    if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
+    then 
+      echo "File modification date is greater than 30 hours"
+      json_content=`cat $slow_query_status_file`
+      message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
+      echo $message 
+      exit 2
+    else
+      echo "File is empty for now"
+    fi
+  fi
+else
+  # Check if the result exists
+  {
+    # get the total number of queries run and the max time
+    # TODO: improve regex
+    # TODO: improve the parameters (currently we are using threshold on queries and max execute time)
+    # # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
+    regex="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
+    file_content=`cat $digest_file`
+    if [[ $file_content =~ $regex ]]
+    then
+      max_query_threshold=${BASH_REMATCH[1]}
+      slowest_query_threshold=${BASH_REMATCH[2]}
+      hasK="${max_query_threshold: -1}"
+      if [[ "$hasK" == "k" ]] 
+      then
+        pre="${max_query_threshold::-1}"
+        max_query_threshold=$(echo "scale=4; ${pre:-0}*1000" | bc)
+      else
+        max_query_threshold=${max_query_threshold:-0}
+      fi
+      # TODO: support ms
+      slowest_query_threshold="${slowest_query_threshold:-0}"
+      if [[ `echo "$max_query_threshold < $desired_max_query_threshold" | bc` -eq 1 && `echo "$slowest_query_threshold > $desired_slow_query_threshold" | bc` -eq 1 ]]
+      then
+        echo "Total number of slow queries are: $max_query_threshold"
+        echo "Time taken by slowest query is: $slowest_query_threshold"
+        echo "Thanks for keeping it all clean"
+      else
+        echo "Ops! One of the two expected parameters did not meet"
+        echo "Time taken by slowest query is: $slowest_query_threshold s and required maximum is $desired_slow_query_threshold s"
+        echo "Total slow queries are $max_query_threshold and expected maximum value is $desired_max_query_threshold"
+        exit 2
+      fi
+    else
+      echo "No threshold found in the result"
+    fi
+  } || {
+    echo "Cannot parse the result"
+  }
+fi

stack/erp5/instance-mariadb.cfg.in

@@ -283,7 +283,7 @@ recipe = slapos.recipe.template:jinja2
 template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
 rendered = ${monitor-directory:reports}/${:filename}
 output-folder = ${directory:srv}/monitor/
-filename = mariadb_slow_query_every_23_hour
+filename = mariadb_slow_query_every_2_minutes
 mode = 755
 context = 
   raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
@@ -291,6 +291,23 @@ context =
   raw dash {{ parameter_dict['dash-location'] }}/bin/dash
   key output_folder :output-folder 
 
+[slowquery-parameters]
+max-queries-threshold = ${monitor-directory:etc}/max-slowqueries-threshold
+slowest-queries-threshold = ${monitor-directory:etc}/slowest-queries-threshold
+
+[{{ section('monitor-promise-slowquery-result') }}]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
+rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
+status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
+context =
+  raw default_threshold 0.7
+  raw bash {{ parameter_dict['bash'] }}/bin/bash
+  raw slow_query_digest ${directory:srv}/monitor
+  key slow_query_status :status-file
+  key max_queries_threshold slowquery-parameters:max-queries-thershold
+  key slowest_queries_threshold slowquery-parameters:slowest-queries-threshold
+
 [{{ section('promise') }}]
 recipe = slapos.cookbook:wrapper
 command-line = "{{ parameter_dict['bin-directory'] }}/is-local-tcp-port-opened" "${my-cnf-parameters:ip}" "${my-cnf-parameters:port}"
@@ -302,6 +319,9 @@ monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
 monitor-httpd-port = {{ port + 1 }}
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
+instance-configuration =
+  file max-queries-threshold ${slowquery-parameteres:max-queries-threshold} 
+  file slowest-queries-threshold ${slowquery-parameters:slowest-queries-threshold}
 
 [buildout]
 extends =

stack/erp5/instance.cfg.in

@@ -93,6 +93,7 @@ openssl-location = {{ openssl_location }}
 
 [dynamic-template-balancer-parameters]
 apache = {{ apache_location }}
+apdex-result-check-script = {{ apdex_result_check_script }}
 openssl = {{ openssl_location }}
 haproxy = {{ haproxy_location }}
 bin-directory = {{ bin_directory }}
@@ -101,6 +102,7 @@ run-apachedex-location = {{ bin_directory }}/runApacheDex
 6tunnel = {{ sixtunnel_location }}
 curl-location = {{ curl_location }}
 dash = {{ dash_location }}
+bash = {{ bash_location }}
 template-haproxy-cfg = {{ template_haproxy_cfg }}
 template-apache-conf = {{ template_apache_conf }}
 template-monitor = {{ dumps(template_monitor) }}
@@ -169,6 +171,7 @@ extra-context =
     section parameter_dict dynamic-template-kumofs-parameters
 
 [dynamic-template-mariadb-parameters]
+bash = {{ bash_location }}
 coreutils-location = {{ coreutils_location }}
 dash-location = {{ dash_location }}
 findutils-location = {{ findutils_location }}
@@ -180,6 +183,7 @@ link-binary = {{ dumps(mariadb_link_binary) }}
 bin-directory = {{ bin_directory }}
 mariadb-resiliency-after-import-script = {{ mariadb_resiliency_after_import_script }}
 mariadb-slow-query-report-script = {{ mariadb_slow_query_report_script }}
+mariadb-slowquery-check-script = {{ mariadb_slowquery_check_script}}
 percona-tools-location = {{ percona_toolkit_location }}
 template-monitor = {{ template_monitor }}
 

stack/erp5/mysql-querydigest.sh.in

@@ -10,6 +10,8 @@ if [ ! -d "$OUTPUT_FOLDER" ]; then
   exit 0
 fi
 
+OUTPUT_FILE=${OUTPUT_FOLDER}/slowquery_digest.txt
+
 TODAY=`date +%Y%m%d`
 
 SLOW_LOG=$SLOW_QUERY_PATH-$TODAY
@@ -19,4 +21,4 @@ if [ ! -f "$SLOW_LOG" ]; then
   exit 1
 fi
 
-eval $PT_QUERY_EXEC $SLOW_LOG
+eval $PT_QUERY_EXEC $SLOW_LOG | tee $OUTPUT_FILE