stack/erp5: Clean up apachedex and slowquery promises

stack/erp5/buildout.hash.cfg

@@ -19,15 +19,15 @@ md5sum = 844d62cd6f9d6e3d1d78d52de2b72a49
 
 [mariadb-slow-query-report-script]
 filename = mysql-querydigest.sh.in
-md5sum = 48fb6a0ed34cfc6de6b40acd7088c51c
+md5sum = cfe6ab8ae54a521ecb269e9d9762cbeb
 
 [mariadb-slowquery-check-script]
 filename = instance-mariadb-check-slowquery-result.sh.in 
-md5sun = c9cd394f3d031cef77da50610544d2b0
+md5sum = 356e0e2db1da0e8b479908fb739e5cc0
 
 [template-mariadb]
 filename = instance-mariadb.cfg.in
-md5sum = 9c34cba998ce0ed3b96e2604d2ed1cb5
+md5sum = aac99b03aa10b60e2b17d6882513b5bd
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
@@ -103,7 +103,7 @@ md5sum = 79fc39f7fbf13b1788adb5c33150dd80
 
 [apdex-result-check-script]
 filename = instance-balancer-check-apachedex-result.sh.in 
-md5sum = e1dd2f2884ea0e9f6f230419e032d927
+md5sum = cdc5e22560a26e072bade314bd4c506e
 
 [template-haproxy-cfg]
 filename = haproxy.cfg.in

stack/erp5/instance-balancer-check-apachedex-result.sh.in

@@ -2,31 +2,25 @@
 
 set -e
 
-apachedex_file='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
-apachedex_status_file={{ apdex_status_file }} 
-desired_threshold={{ default_threshold }}
-user_threshold=$(<{{ user_threshold }})
-
-if [ ! -z "$user_threshold" ]
-then
-  desired_threshold=$user_threshold
-fi
+APACHEDEX_FILE='{{ apdex_file }}/ApacheDex-'$(date +%Y-%m-%d)'.html'
+APACHEDEX_REPORT_JSON_FILE={{ apdex_status_file }} 
+DESIRED_THRESHOLD={{ user_threshold }}
 
 # Check if the file is there
-if [ ! -s "$apachedex_file" ]; then 
+if [ ! -s "$APACHEDEX_FILE" ]; then 
   # If file doesn't exists create one
   # If it is empty check for modification time
-  if [ ! -f "$apachedex_file" ]; then
-    touch $apachedex_file
+  if [ ! -f "$APACHEDEX_FILE" ]; then
+    touch $APACHEDEX_FILE
   else
-    modified_date=`stat -c '%Z' $apachedex_file`
-    current_date=`date +%s`
-    if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
+    MODIFIED_DATE=`stat -c '%Z' $APACHEDEX_FILE`
+    CURRENT_DATE=`date +%s`
+    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
     then 
       echo "File modification date is greater than 30 hours"
-      json_content=`cat $apachedex_status_file`
-      message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $message 
+      JSON_CONTENT=`cat $APACHEDEX_REPORT_STATUS_FILE`
+      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
+      echo $MESSAGE
       exit 2
     else
       echo "File is empty for now"
@@ -35,17 +29,17 @@ if [ ! -s "$apachedex_file" ]; then
 else
   # Check if the result exists
   {
-    regex="apdex threshold<\/th><td>(.*)s<\/td>"
-    file_content=`cat $apachedex_file`
-    if [[ $file_content =~ $regex ]]
+    REGEX="<th>apdex.*?<tr><td [^<]*>(.*?)%<\/td>"
+    FILE_CONTENT=`cat $APACHDEX_FILE`
+    if [[ $FILE_CONTENT =~ $REGEX ]]
     then
-      threshold=${BASH_REMATCH[1]}
-      threshold=${threshold:-0}
-      if [[ `echo "$threshold > $desired_threshold" | bc` -eq 1 ]]
+      RESULT=${BASH_REMATCH[1]}
+      RESULT=${THRESHOLD:-0}
+      if [[ `echo "$RESULT > $DESIRED_THRESHOLD" | bc` -eq 1 ]]
       then
-        echo "Your score is $threshold, Thanks for keeping it all clean"
+        echo "Your score is $RESULT %, Thanks for keeping it all clean"
       else
-        echo "Threshold is lower than exptected: Expected was $desired_threshold and we current is $threshold"
+        echo "Threshold is lower than exptected: Expected was $DESIRED_THRESHOLD % and we current is $RESULT %"
         exit 2
       fi
     else

stack/erp5/instance-mariadb-check-slowquery-result.sh.in

 #!{{ bash }}
 set -e
 
-digest_file='{{ slow_query_digest }}/slowquery_digest.txt'
-slow_query_status_file='{{ slow_query_status }}'
-desired_max_query_threshold={{ default_threshold }}
-desired_slow_query_threshold=0
-
-max_queries_threshold='{{ max_queries_threshold  }}'
-if [ ! -z "$max_queries_threshold" ]
-then
-  desired_max_query_threshold=$max_queries_threshold
-fi
-
-slowest_queries_threshold='{{ slowest_queries_threshold  }}'
-if [ ! -z "$slowest_queries_threshold" ]
-then
-  desired_slow_query_threshold=$slowest_queries_threshold
-fi
+DIGEST_FILE='{{ slow_query_digest }}/slowquery_digest.txt'
+SLOW_QUERY_STATUS_FILE='{{ slow_query_status }}'
+DESIRED_MAX_QUERY_THRESHOLD={{ max_queries_threshold }}
+DESIRED_SLOW_QUERY_THRESHOLD={{ slowest_queries_threshold  }}
 
 # Check if the file is there
-if [ ! -s "$digest_file" ]; then 
+if [ ! -s "$DIGEST_FILE" ]; then 
   # If file doesn't exists create one
   # If it is empty check for modification time
-  if [ ! -f "$digest_file" ]; then
-    touch $digest_file
+  if [ ! -f "$DIGEST_FILE" ]; then
+    touch $DIGEST_FILE
   else
-    modified_date=`stat -c '%Z' $digest_file`
-    current_date=`date +%s`
-    if [[ `echo "$current_date - $modified_date" | bc` -gt 108000 ]]
+    MODIFIED_DATE=`stat -c '%Z' $DIGEST_FILE`
+    CURRENT_DATE=`date +%s`
+    if [[ `echo "$CURRENT_DATE - $MODIFIED_DATE" | bc` -gt 108000 ]]
     then 
       echo "File modification date is greater than 30 hours"
-      json_content=`cat $slow_query_status_file`
-      message=`echo $json_content | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
-      echo $message 
+      JSON_CONTENT=`cat $SLOW_QUERY_STATUS_FILE`
+      MESSAGE=`echo $JSON_CONTENT | python -c 'import json,sys;obj=json.load(sys.stdin);print obj["message"]'`
+      echo $MESSAGE 
       exit 2
     else
       echo "File is empty for now"
@@ -41,35 +29,35 @@ if [ ! -s "$digest_file" ]; then
 else
   # Check if the result exists
   {
-    # get the total number of queries run and the max time
+    # get the total number of queries ran and the max time
     # TODO: improve regex
     # TODO: improve the parameters (currently we are using threshold on queries and max execute time)
     # # Overall: (.*) total,(?:.*\n){4}# Exec time(?: *\d*m?s){2} *(.*?)m?s
-    regex="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
-    file_content=`cat $digest_file`
-    if [[ $file_content =~ $regex ]]
+    REGEX="# Overall: (.*) total,.*# Exec time *[[:digit:]]*m?s *[[:digit:]]*m?s *([[:digit:]]*)m?s"
+    FILE_CONTENT=`cat $DIGEST_FILE`
+    if [[ $FILE_CONTENT =~ $REGEX ]]
     then
-      max_query_threshold=${BASH_REMATCH[1]}
-      slowest_query_threshold=${BASH_REMATCH[2]}
-      hasK="${max_query_threshold: -1}"
-      if [[ "$hasK" == "k" ]] 
+      TOTAL_QUERIES_EXEC=${BASH_REMATCH[1]}
+      SLOWEST_QUERY_TIME=${BASH_REMATCH[2]}
+      HAS_K="${TOTAL_QUERIES_EXEC: -1}"
+      if [[ "$HAS_K" == "k" ]] 
       then
-        pre="${max_query_threshold::-1}"
-        max_query_threshold=$(echo "scale=4; ${pre:-0}*1000" | bc)
+        PRE="${TOTAL_QUERIES_EXEC::-1}"
+        TOTAL_QUERIES_EXEC=$(echo "scale=4; ${PRE:-0}*1000" | bc)
       else
-        max_query_threshold=${max_query_threshold:-0}
+        TOTAL_QUERIES_EXEC=${TOTAL_QUERIES_EXEC:-0}
       fi
       # TODO: support ms
-      slowest_query_threshold="${slowest_query_threshold:-0}"
-      if [[ `echo "$max_query_threshold < $desired_max_query_threshold" | bc` -eq 1 && `echo "$slowest_query_threshold > $desired_slow_query_threshold" | bc` -eq 1 ]]
+      SLOWEST_QUERY_TIME="${SLOWEST_QUERY_TIME:-0}"
+      if [[ `echo "$TOTAL_QUERIES_EXEC < $DESIRED_MAX_QUERY_THRESHOLD" | bc` -eq 1 && `echo "$SLOWEST_QUERY_TIME < $DESIRED_SLOW_QUERY_THRESHOLD" | bc` -eq 1 ]]
       then
-        echo "Total number of slow queries are: $max_query_threshold"
-        echo "Time taken by slowest query is: $slowest_query_threshold"
+        echo "Total number of slow queries are: $TOTAL_QUERIES_EXEC"
+        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME"
         echo "Thanks for keeping it all clean"
       else
         echo "Ops! One of the two expected parameters did not meet"
-        echo "Time taken by slowest query is: $slowest_query_threshold s and required maximum is $desired_slow_query_threshold s"
-        echo "Total slow queries are $max_query_threshold and expected maximum value is $desired_max_query_threshold"
+        echo "Time taken by slowest query is: $SLOWEST_QUERY_TIME s and required maximum is $DESIRED_SLOW_QUERY_THRESHOLD s"
+        echo "Total slow queries are $TOTAL_QUERIES_EXEC and expected maximum value is $DESIRED_MAX_QUERY_THRESHOLD"
         exit 2
       fi
     else

stack/erp5/instance-mariadb.cfg.in

@@ -282,14 +282,13 @@ context =
 recipe = slapos.recipe.template:jinja2
 template = {{ parameter_dict['mariadb-slow-query-report-script'] }}
 rendered = ${monitor-directory:reports}/${:filename}
-output-folder = ${directory:srv}/monitor/
-filename = mariadb_slow_query_every_2_minutes
+filename = mariadb_slow_query_every_23_hour
 mode = 755
 context = 
   raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
   raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
   raw dash {{ parameter_dict['dash-location'] }}/bin/dash
-  key output_folder :output-folder 
+  key output_folder monitor-directory:private
 
 [slowquery-parameters]
 max-queries-threshold = ${monitor-directory:etc}/max-slowqueries-threshold
@@ -301,11 +300,11 @@ template = {{ parameter_dict['mariadb-slowquery-check-script'] }}
 rendered = ${monitor-directory:promises}/mariadb-slow-queries-result
 status-file = ${monitor-directory:private}/mariadb_slow_query.report.json
 context =
-  raw default_threshold 0.7
+  raw default_threshold 4000
   raw bash {{ parameter_dict['bash'] }}/bin/bash
-  raw slow_query_digest ${directory:srv}/monitor
+  key slow_query_digest monitor-directory:private
   key slow_query_status :status-file
-  key max_queries_threshold slowquery-parameters:max-queries-thershold
+  key max_queries_threshold slowquery-parameters:max-queries-threshold
   key slowest_queries_threshold slowquery-parameters:slowest-queries-threshold
 
 [{{ section('promise') }}]
@@ -320,7 +319,7 @@ monitor-httpd-port = {{ port + 1 }}
 monitor-title = {{ slapparameter_dict['name'] }}
 password = {{ slapparameter_dict['monitor-passwd'] }}
 instance-configuration =
-  file max-queries-threshold ${slowquery-parameteres:max-queries-threshold} 
+  file max-queries-threshold ${slowquery-parameters:max-queries-threshold} 
   file slowest-queries-threshold ${slowquery-parameters:slowest-queries-threshold}
 
 [buildout]

stack/erp5/mysql-querydigest.sh.in

@@ -21,4 +21,5 @@ if [ ! -f "$SLOW_LOG" ]; then
   exit 1
 fi
 
-eval $PT_QUERY_EXEC $SLOW_LOG | tee $OUTPUT_FILE
+$PT_QUERY_EXEC $SLOW_LOG >  $OUTPUT_FILE
+echo "ok"