Added notebook for visualization of data

bt5/erp5_wendelin_drone/PathTemplateItem/notebook_module/1.xml

+<?xml version="1.0"?>
+<ZopeData>
+  <record id="1" aka="AAAAAAAAAAE=">
+    <pickle>
+      <global name="Notebook" module="erp5.portal_type"/>
+    </pickle>
+    <pickle>
+      <dictionary>
+        <item>
+            <key> <string>_Access_contents_information_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Manager</string>
+                <string>Owner</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Add_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Manager</string>
+                <string>Owner</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Change_local_roles_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignor</string>
+                <string>Manager</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_Modify_portal_content_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Manager</string>
+                <string>Owner</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>_View_Permission</string> </key>
+            <value>
+              <tuple>
+                <string>Assignee</string>
+                <string>Assignor</string>
+                <string>Manager</string>
+                <string>Owner</string>
+              </tuple>
+            </value>
+        </item>
+        <item>
+            <key> <string>content_md5</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>description</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>id</string> </key>
+            <value> <string>1</string> </value>
+        </item>
+        <item>
+            <key> <string>short_title</string> </key>
+            <value>
+              <none/>
+            </value>
+        </item>
+        <item>
+            <key> <string>text_content</string> </key>
+            <value> <string encoding="cdata"><![CDATA[
+
+%% md\n
+# Drone Simulation Results (best current simulation)\n
+<div class="plot_div" id="plot_div"/>\n
+\n
+\n
+%% fetch\n
+js: jio.js\n
+js: ndarray_bundle.js\n
+js: wendelin.js\n
+js: https://cdn.plot.ly/plotly-latest.min.js\n
+js: https://cdn.jsdelivr.net/pyodide/v0.23.4/full/pyodide.js\n
+\n
+%% js\n
+hateoas_url = "https://softinst195526.host.vifib.net/erp5/web_site_module/renderjs_runner/hateoas/";\n
+jio = jIO.createJIO({\n
+  type: "erp5",\n
+  url: hateoas_url,\n
+  default_view_reference: "view"\n
+});\n
+\n
+gadget = {\n
+  getSetting: function(property) {\n
+    return new RSVP.Queue()\n
+      .push(function () {\n
+        if (property == "hateoas_url") {\n
+          return hateoas_url;\n
+        }\n
+        return;\n
+      });\n
+  },\n
+  jio_getAttachment: function(id, url, parameter_dict) {\n
+    return jio.getAttachment(id, url, parameter_dict);\n
+  },\n
+  jio_get: function(id) {return jio.get(id);}\n
+}\n
+\n
+\n
+var graph = document.getElementById(\'plot_div\'),\n
+graph_data = [],\n
+label_list = ["ASML",\t"climb_rate",\t"distance",\t"ground_speed",\t"time"]; \n
+\n
+\n
+\n
+\n
+function unpack(rows, key) {\n
+return rows.map(function(row) { return row[key]; });\n
+}\n
+\n
+// TODO we still need to manually type in the index range for the iteration we are interested in, because for some reason the data array lines are not ploted\n
+function getData(){\n
+var graph_data=[],\n
+ratio = screen.availWidth / 1024,\n
+interval,\n
+frequency,\n
+start_index = 0,\n
+stop_index = 10000,\n
+array_id;\n
+\n
+\n
+return jio.allDocs({\n
+query: \'portal_type:"Data Array" AND \' +\n
+\'title: "Create Plot Datas"\'\n
+})\n
+.push(function (result) {\n
+array_id = result.data.rows[0].id;\n
+return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);\n
+})\n
+.push(function(result) {\n
+for (i = 0; i < label_list.length; i += 1) {\n
+graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));\n
+}\n
+\n
+return graph_data\n
+})\n
+.push(function (graph_data) {\n
+var trace1 = {\n
+        type: "scattergl",\n
+        mode: "markers",\n
+        name: "distance",\n
+        x: unpack(graph_data, label_list.indexOf("time")),\n
+        y: unpack(graph_data, label_list.indexOf("distance")),\n
+        line: {color: \'#1f77b4\'}\n
+        };\n
+        \n
+var trace2 = {\n
+        type: "scattergl",\n
+        mode: "markers",\n
+        name: \'ASML\',\n
+        x: unpack(graph_data, label_list.indexOf("time")),\n
+        y: unpack(graph_data, label_list.indexOf("ASML")),\n
+        line: {color: \'#ff7f0e\'}\n
+        }\n
+        \n
+var data = [trace1, trace2];\n
+return data\n
+});\n
+}\n
+\n
+\n
+function plot () {\n
+return getData()\n
+.push(function (data) {\n
+Plotly.newPlot(\n
+graph,\n
+data,\n
+    { \'title\' :\'Distance and ASML over time\'\n
+    });\n
+    graph.on(\'plotly_relayout\', function(eventdata) {\n
+      var start_date = new Date(eventdata["xaxis.range[0]"]),\n
+      stop_date = new Date(eventdata["xaxis.range[1]"]);\n
+      return update_graph(start_date, stop_date);\n
+    });\n
+  });\n
+}\n
+\n
+\n
+plot();\n
+\n
+%% md\n
+<div class="plot_div2" id="plot_div2"/>\n
+\n
+\n
+%% js\n
+\n
+var graph = document.getElementById(\'plot_div2\');\n
+function getData(){\n
+var graph_data=[],\n
+ratio = screen.availWidth / 1024,\n
+interval,\n
+frequency,\n
+start_index,\n
+stop_index,\n
+array_id;\n
+\n
+return jio.allDocs({\n
+query: \'portal_type:"Data Array" AND \' +\n
+\'title: "Create Plot Datas"\'\n
+})\n
+.push(function (result) {\n
+array_id = result.data.rows[0].id;\n
+console.log(array_id);\n
+return wendelin.getArrayRawSlice(gadget, array_id, 0, 1);\n
+})\n
+.push(function (result) {\n
+start_index = 0,\n
+stop_index = 100000;\n
+\n
+return wendelin.getArrayRawSlice(gadget, array_id, start_index, stop_index);\n
+})\n
+.push(function(result) {\n
+for (i = 0; i < label_list.length; i += 1) {\n
+graph_data = graph_data.concat(nj.unpack(result.pick( null, label_list[i])));\n
+}\n
+return graph_data\n
+})\n
+.push(function (graph_data) {\n
+\n
+var trace1 = {\n
+        type: "scatter",\n
+        mode: "markers",\n
+        name: "ground speed",\n
+        x: unpack(graph_data, label_list.indexOf("time")),\n
+        y: unpack(graph_data, label_list.indexOf("ground_speed")),\n
+        line: {color: \'#1f77b4\'}\n
+        };\n
+        \n
+var trace2 = {\n
+        type: "scatter",\n
+        mode: "markers",\n
+        name: \'climb rate\',\n
+        x: unpack(graph_data, label_list.indexOf("time")),\n
+        y: unpack(graph_data, label_list.indexOf("climb_rate")),\n
+        line: {color: \'#ff7f0e\'}\n
+        }\n
+        \n
+var data = [trace1, trace2];\n
+return data\n
+});\n
+}\n
+\n
+\n
+\n
+function plot () {\n
+return getData()\n
+.push(function (data) {\n
+Plotly.newPlot(\n
+graph,\n
+data,\n
+    { \'title\' :\'Ground speed and climb rate over time\'\n
+    });\n
+    graph.on(\'plotly_relayout\', function(eventdata) {\n
+      var start_date = new Date(eventdata["xaxis.range[0]"]),\n
+      stop_date = new Date(eventdata["xaxis.range[1]"]);\n
+      return update_graph(start_date, stop_date);\n
+    });\n
+  });\n
+}\n
+\n
+\n
+plot();
+
+]]></string> </value>
+        </item>
+        <item>
+            <key> <string>title</string> </key>
+            <value> <string>DroneSim</string> </value>
+        </item>
+      </dictionary>
+    </pickle>
+  </record>
+</ZopeData>

bt5/erp5_wendelin_drone/PathTemplateItem/portal_callables/fitness_score_drones.py

@@ -46,23 +46,29 @@ def create_scorelist(scores):
 
 
 
-def create_score_dataframe(scores, mean_lists, iteration=0):
-    #df = pd.DataFrame(columns=['Iteration', 'Sim Name', 'Parameters', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
-    df = pd.DataFrame(columns=['Iteration', 'Parameters1', 'Parameters2', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
+def create_score_dataframe(scores, mean_lists):
+    df = pd.DataFrame(columns=['Sim_name', 'Parameters1', 'Parameters2', 'Score', 'Distance mean', 'ASML mean', 'Ground Speed mean', 'Climb Rate mean'])
     for score in scores:
-        row=[int(iteration)]
-        row.extend([score[2][0],score[2][1], int(score[3])])
+        row=[]
+        row.extend([score[1],score[2][0],score[2][1], int(score[3])])
         for data in mean_lists:
             for i in data:
-                #print(i)
                 if i[1] == score[0]:
                     row.append(i[0])
                     break
-        context.log(row)
         new_row = pd.DataFrame([row], columns=df.columns)
         df = pd.concat([df, new_row], ignore_index=True)
     return df
 
+def create_plot_dataframe(plot_data):
+  df = pd.DataFrame({
+    "time": plot_data[0],
+    "distance":plot_data[1],
+    "ASML": plot_data[2],
+    "ground_speed": plot_data[3],
+    "climb_rate": plot_data[4]
+    })
+  return df
 
 real_array = input_array_real["Data Array"]
 sim_array = input_array_sim["Data Array"]
@@ -179,21 +185,13 @@ timestamp_list = simulated_flights_value_dict_list[to_plot_index]["timestamp"]
 score_list2 = create_scorelist(sorted_score_list)
 
 
-
-
 zbigarray = out_array_scores["Data Array"].getArray()
 
-if zbigarray is None:
-  iteration = 1
-else:
-  try:
-    iteration = zbigarray[-1][1] + 1 # Look at the last iteration number
-  except:
-    iteration = 1
+score_names = list(out_array_scores["Data Array"])
 
 
-df = create_score_dataframe(score_list2, mean_list_list, iteration)
-dtypes = {'index': 'i8', 'Iteration': 'i8', 'Parameters1': 'f8', 'Parameters2': 'f8',
+df = create_score_dataframe(score_list2, mean_list_list)
+dtypes = {'index': 'i8','Sim_name':'S256', 'Parameters1': 'f16', 'Parameters2': 'f16',
           'Score': 'f8', 'Distance_mean': 'f8', 'ASML_mean': 'f8',
           'Ground_Speed_mean': 'f8', 'Climb_Rate_mean': 'f8'}
 
@@ -205,10 +203,105 @@ ndarray = df.to_records(column_dtypes=dtypes, index = False)
 if zbigarray is None:
   zbigarray = out_array_scores["Data Array"].initArray(shape=(0,), dtype=ndarray.dtype.fields)
 
+score_array_start_idx = zbigarray.shape[0]
 
 zbigarray.append(ndarray)
 
 
+
+new_key = None
+
+try:
+  max_nr = 0
+  for key in score_names:
+    old_key_nr = int(key.split("_")[1])
+    if old_key_nr>max_nr:
+      max_nr=old_key_nr
+    new_key = "Iteration_" + str(max_nr+1)
+except:
+  new_key = "Iteration_1"
+
+
+
+
+
+data_array_line_score = out_array_scores.get(new_key)
+
+if data_array_line_score is None:
+  data_array_line_score = out_array_scores["Data Array"].newContent(id=new_key,
+                                             portal_type="Data Array Line")
+
+data_array_line_score.edit(reference=new_key,
+     index_expression="%s:%s" %(score_array_start_idx, zbigarray.shape[0])
+  )
+
+
+
+zbigarray2 = out_array_plot_data["Data Array"].getArray()
+#Uncomment this to set the array to empty
+#out_array_plot_data["Data Array"] = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=ndarray.dtype.fields)
+
+things = [timestamp_list,
+    simulated_flights_value_dict_list[to_plot_index]["distance"],
+    simulated_flights_value_dict_list[to_plot_index]["ASML"],
+    simulated_flights_value_dict_list[to_plot_index]["ground speed"],
+    simulated_flights_value_dict_list[to_plot_index]["climb rate"]]
+
+
+
+
+
+
+
+df = create_plot_dataframe(things)
+dtypes = {'time': 'f8', 'distance': 'f8', 'ASML': 'f8',
+          'ground_speed': 'f8', 'climb_rate': 'f8'}
+
+
+ndarray2 = df.to_records(column_dtypes=dtypes, index = False)
+context.log(ndarray2)
+if zbigarray2 is None:
+  context.log("zbigarray2 is none")
+  zbigarray2 = out_array_plot_data["Data Array"].initArray(shape=(0,), dtype=ndarray2.dtype.fields)
+
+
+plot_array_start_idx = zbigarray2.shape[0]
+
+zbigarray2.append(ndarray2)
+
+
+
+new_key_plot = "PlotIteration_1"
+plot_names = list(out_array_plot_data["Data Array"])
+context.log(plot_names)
+try:
+  context.log("OK")
+  max_nr = 0
+  for key in plot_names:
+    old_key_nr = int(key.split("_")[1])
+    if old_key_nr>max_nr:
+      max_nr=old_key_nr
+    new_key_plot = "PlotIteration_" + str(max_nr+1)
+    
+except:
+  context.log("h")
+  new_key_plot = "PlotIteration_1"
+
+context.log(new_key_plot)
+
+
+
+
+data_array_line_plot = out_array_plot_data.get(new_key_plot)
+
+if data_array_line_plot is None:
+  data_array_line_plot = out_array_plot_data["Data Array"].newContent(id=new_key_plot,
+                                             portal_type="Data Array Line")
+
+data_array_line_plot.edit(reference=new_key_plot,
+     index_expression="%s:%s" %(plot_array_start_idx, zbigarray2.shape[0])
+  )
+
 if end > start:
   progress_indicator_sim.setIntOffsetIndex(end)
 

bt5/erp5_wendelin_drone/bt/description

-This business template shows how to ingest the bucket stream data we get from the drone simulations.
\ No newline at end of file
+Ingest simulation drone data, score the simulations and display some relevant information.
\ No newline at end of file

bt5/erp5_wendelin_drone/bt/template_keep_path_list

@@ -12,6 +12,7 @@ data_supply_module/4
 data_transformation_module/4
 data_transformation_module/5
 data_transformation_module/6
+notebook_module/1
 portal_callables/DataIngestionline_writeFluentdIngestionToDataBucketStream
 portal_callables/convert_log_array_script
 portal_callables/fitness_score_drones
\ No newline at end of file

bt5/erp5_wendelin_drone/bt/template_path_list

@@ -12,6 +12,7 @@ data_supply_module/4
 data_transformation_module/4
 data_transformation_module/5
 data_transformation_module/6
+notebook_module/1
 portal_callables/DataIngestionline_writeFluentdIngestionToDataBucketStream
 portal_callables/convert_log_array_script
 portal_callables/fitness_score_drones
\ No newline at end of file