Improved generation of HTML file in px4_flight_review action

Context:
The px4_flight_review action generates an HTML file using the
underlying PX4/flight_review code. This code is able to generate
an HTML file including a flight map, Bokeh plots and log tables.
However, the current action implementation generated an HTML file
that only included the Bokeh plots.

Solution:
Refactor how the action generates an HTML file, by better utilizing
functions available in Bokeh as well as a Jinja template to lay out
the resulting page with map and tables included. The map required
inclusion of leaflet and mapbox script dependencies as well as an
API token that is now exposed as an action parameter so it can be
overridden by end users.

Testing:
Verified behavior by running the ./build.sh and ./test.sh scripts.
Deployed action to Roboto dev stack account and ran with a variety
of PX4 ULog files. Observed improved HTML artifacts that are much
closer to the public PX4 flight review equivalents.

.gitignore

@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# OS X
+.DS_Store

actions/px4_flight_review/Dockerfile

@@ -8,7 +8,6 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-
 # Install dependencies required for Python 3.10
 RUN apt-get update && \
     apt-get install -y software-properties-common && \
@@ -26,24 +25,6 @@ RUN apt-get update && apt-get install -y \
     xvfb \
     && rm -rf /var/lib/apt/lists/*
 
-# Define the Firefox and geckodriver versions
-ENV FIREFOX_VERSION 92.0
-ENV GECKODRIVER_VERSION 0.31.0
-
-# Install Firefox
-RUN wget --no-verbose -O /tmp/firefox.tar.bz2 \
-    https://ftp.mozilla.org/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 \
-    && tar -xjf /tmp/firefox.tar.bz2 -C /opt/ \
-    && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-    && rm /tmp/firefox.tar.bz2
-
-# Install geckodriver
-RUN wget --no-verbose -O /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz \
-    && tar -xzf /tmp/geckodriver.tar.gz -C /opt/ \
-    && ln -s /opt/geckodriver /usr/local/bin/geckodriver \
-    && rm /tmp/geckodriver.tar.gz
-
 # Upgrade pip
 RUN python3.10 -m pip install --upgrade pip
 

actions/px4_flight_review/action.json

@@ -9,10 +9,10 @@
     },
     "parameters": [
         {
-            "name": "SAVE_PDF",
+            "name": "MAPBOX_API_TOKEN",
             "required": false,
-            "description": "Set True to save a .pdf",
-            "default": "False"
+            "description": "Mapbox API Access Token",
+            "default": "pk.eyJ1IjoiYmt1ZW5nIiwiYSI6ImNqb3p2Mjl3ZjAwbDAzdm51YTIxdTBra3kifQ.aEC7iOQQYMshIzS_vTxtlA"
         }
     ],
     "tags": ["px4"],

actions/px4_flight_review/requirements.runtime.txt

@@ -1,8 +1,2 @@
 # Python packages to install within the Docker image associated with this Action.
 roboto
-#bokeh==3.0.0
-pyulog
-scipy
-pyfftw
-img2pdf
-selenium

actions/px4_flight_review/scripts/test.sh

@@ -77,11 +77,6 @@ main() {
     run_docker_test ""
     file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.html
 
-    # Test 2
-     echo "Running Test 2: Test .pdf report generation"
-     clean_actual_output
-     run_docker_test "-e ROBOTO_PARAM_SAVE_PDF=True"
-     file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.pdf
 }
 
 # Run the main test execution

actions/px4_flight_review/src/px4_flight_review/__main__.py

@@ -1,109 +1,90 @@
 import argparse
 import os
+import io
 import pathlib
-from bokeh.plotting import output_file, save
+from jinja2 import Template
 from helper import load_ulog_file
 from pyulog.px4 import PX4ULog
 from db_entry import DBData
 from configured_plots import generate_plots
-import re
-from bokeh.io import export_png
-import img2pdf
+from plotted_tables import get_heading_html
+from bokeh.layouts import column
+from bokeh.resources import CDN
+from bokeh.embed import components
+from bokeh.io import curdoc
 from roboto.domain import actions
 
-
-def extract_plot_html(file_path):
-    """Extracts the HTML content of a Bokeh plot from a saved file."""
-    with open(file_path, 'r') as file:
-        content = file.read()
-        plot_html = re.search(r'(?<=<body>).*(?=</body>)', content, re.DOTALL).group()
-    return plot_html
-
-
-def extract_head_html(file_path):
-    """Extracts the HTML content for Bokeh resources from the <head> section."""
-    with open(file_path, 'r') as file:
-        content = file.read()
-        head_html = re.search(r'(?<=<head>).*(?=</head>)', content, re.DOTALL).group()
-    return head_html
-
-
-def process_ulg_file(ulog_file_path, output_folder, save_pdf):
+def process_ulg_file(ulog_file_path, output_folder):
     """Process a single .ulg file to generate and combine Bokeh plots into an HTML report."""
     ulog = load_ulog_file(ulog_file_path)
     px4_ulog = PX4ULog(ulog)
     db_data = DBData()
-    vehicle_data = None  # You will need to determine how to set this
-
-    log_id = "av"
-    link_to_3d_page = '3d?log=' + log_id
-    link_to_pid_analysis_page = '?plots=pid_analysis&log=' + log_id
-
-    plots = generate_plots(ulog, px4_ulog, db_data, vehicle_data, link_to_3d_page, link_to_pid_analysis_page)
+    vehicle_data = None
+
+    plots = generate_plots(
+        ulog,
+        px4_ulog,
+        db_data,
+        vehicle_data,
+        None,
+        "",
+    )
 
     # Ensure the output directory exists
     output_folder.mkdir(parents=True, exist_ok=True)
-    report_filename = pathlib.Path(ulog_file_path).stem + '_report.html'
+    report_filename = pathlib.Path(ulog_file_path).stem + "_report.html"
     report_filepath = output_folder / report_filename
 
-    # Save individual plots as PNG and create HTML content
-    combined_html_content = ""
-    png_files = []  # List to store paths of PNG files
-
-    for i, plot in enumerate(plots):
-        try:
-            if save_pdf:
-            # Export plot as PNG
-                png_filename = report_filepath.with_suffix(f'.{i}.png')
-                export_png(plot, filename=png_filename)
-                png_files.append(png_filename)
-
-            # Create HTML content (optional if you only need the PDF)
-            plot_file = report_filepath.with_suffix(f'.{i}.html')
-            output_file(plot_file)
-            save(plot)
-
-            if i == 0:  # Extract head from first plot
-                combined_html_content += f"<html><head>{extract_head_html(plot_file)}</head><body>"
-            combined_html_content += extract_plot_html(plot_file)
-
-            # Delete the individual plot file
-            plot_file.unlink(missing_ok=True)
-        except Exception as e:
-            print(f"Error processing plot {i}: {e}")
-
-    combined_html_content += "</body></html>"
-
-    # Save as HTML (optional if you only need the PDF)
-    with open(report_filepath, 'w') as file:
-        file.write(combined_html_content)
-    print(f"Report generated: {report_filepath}")
-
-    if save_pdf:
-        # Combine PNG files into a single PDF
-        pdf_filename = report_filepath.with_suffix('.pdf')
-        with open(pdf_filename, "wb") as f:
-            f.write(img2pdf.convert(png_files))
-        print(f"PDF report generated: {pdf_filename}")
-
-        # Optional: Cleanup PNG files after PDF creation
-        for png_file in png_files:
-            png_file.unlink()
+    # Load Jinja HTML template for summary and plots
+    dirname = os.path.dirname(__file__)
+    j2_template = os.path.join(dirname, 'index.html')
+
+    with open(j2_template) as f:
+        template = Template(f.read())
+
+    # Arrange all generated plots into a column
+    layout = column(plots)
+
+    # Extract Bokeh HTML components so we can lay out our own file
+    # instead of relying on Bokeh's entirely self-generated HTML
+    script_bokeh, div_bokeh = components(layout)
+    resources_bokeh = CDN.render()
+
+    # Compose HTML file with bokeh plots and additional summary data
+    html = template.render(
+        resources=resources_bokeh,
+        script=script_bokeh,
+        plots=div_bokeh,
+        title_html=get_heading_html(ulog, px4_ulog, db_data, None),
+        hardfault_html=curdoc().template_variables.get("hardfault_html"),
+        corrupt_log_html=curdoc().template_variables.get("corrupt_log_html"),
+        error_labels_html=curdoc().template_variables.get("error_labels_html"),
+        info_table_html=curdoc().template_variables.get("info_table_html"),
+        has_position_data=curdoc().template_variables.get("has_position_data"),
+        mapbox_api_access_token=os.environ.get("ROBOTO_PARAM_MAPBOX_API_TOKEN"),
+        pos_datas=curdoc().template_variables.get("pos_datas"),
+        pos_flight_modes=curdoc().template_variables.get("pos_flight_modes"),
+    )
 
+    # Save the HTML file
+    with io.open(report_filepath, mode="w") as f:
+        f.write(html)
 
-def process_ulg_files(input_folder, output_folder, save_pdf):
+def process_ulg_files(input_folder, output_folder):
     """Process all .ulg files within a given folder and its subdirectories."""
     for root, dirs, files in os.walk(input_folder):
         for file in files:
             if file.endswith(".ulg"):
                 full_path = os.path.join(root, file)
                 relative_path = pathlib.Path(root).relative_to(input_folder)
                 output_path = output_folder / relative_path
-                process_ulg_file(full_path, output_path, save_pdf)
+                process_ulg_file(full_path, output_path)
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Generate Bokeh plot reports from ULG files.')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate Bokeh plot reports from ULG files."
+    )
     parser.add_argument(
         "-i",
         "--input-dir",
@@ -123,15 +104,6 @@ def process_ulg_files(input_folder, output_folder, save_pdf):
         default=os.environ.get(actions.InvocationEnvVar.OutputDir.value),
     )
 
-    parser.add_argument(
-        "--save-pdf",
-        action="store_true",
-        required=False,
-        help="Set True to save PDF reports",
-        default=(os.environ.get("ROBOTO_PARAM_SAVE_PDF") == "True"),
-    )
-
     args = parser.parse_args()
 
-    process_ulg_files(args.input_dir, args.output_dir, args.save_pdf)
-
+    process_ulg_files(args.input_dir, args.output_dir)