Merge pull request #1 from roboto-ai/feature/enhancedFlightReview

Improved generation of HTML file in px4_flight_review action
roboto-ai · Jan 23, 2024 · e941e1c · e941e1c
2 parents b95c371 + fac0253
commit e941e1c
Show file tree

Hide file tree

Showing 7 changed files with 253 additions and 120 deletions.
diff --git a/.gitignore b/.gitignore
@@ -158,3 +158,6 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# OS X
+.DS_Store
diff --git a/actions/px4_flight_review/Dockerfile b/actions/px4_flight_review/Dockerfile
@@ -8,7 +8,6 @@ ENV DEBIAN_FRONTEND noninteractive
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-
 # Install dependencies required for Python 3.10
 RUN apt-get update && \
     apt-get install -y software-properties-common && \
@@ -26,24 +25,6 @@ RUN apt-get update && apt-get install -y \
     xvfb \
     && rm -rf /var/lib/apt/lists/*
 
-# Define the Firefox and geckodriver versions
-ENV FIREFOX_VERSION 92.0
-ENV GECKODRIVER_VERSION 0.31.0
-
-# Install Firefox
-RUN wget --no-verbose -O /tmp/firefox.tar.bz2 \
-    https://ftp.mozilla.org/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 \
-    && tar -xjf /tmp/firefox.tar.bz2 -C /opt/ \
-    && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-    && rm /tmp/firefox.tar.bz2
-
-# Install geckodriver
-RUN wget --no-verbose -O /tmp/geckodriver.tar.gz \
-    https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz \
-    && tar -xzf /tmp/geckodriver.tar.gz -C /opt/ \
-    && ln -s /opt/geckodriver /usr/local/bin/geckodriver \
-    && rm /tmp/geckodriver.tar.gz
-
 # Upgrade pip
 RUN python3.10 -m pip install --upgrade pip
 

diff --git a/actions/px4_flight_review/action.json b/actions/px4_flight_review/action.json
@@ -9,10 +9,10 @@
     },
     "parameters": [
         {
-            "name": "SAVE_PDF",
+            "name": "MAPBOX_API_TOKEN",
             "required": false,
-            "description": "Set True to save a .pdf",
-            "default": "False"
+            "description": "Mapbox API Access Token",
+            "default": "pk.eyJ1IjoiYmt1ZW5nIiwiYSI6ImNqb3p2Mjl3ZjAwbDAzdm51YTIxdTBra3kifQ.aEC7iOQQYMshIzS_vTxtlA"
         }
     ],
     "tags": ["px4"],

diff --git a/actions/px4_flight_review/requirements.runtime.txt b/actions/px4_flight_review/requirements.runtime.txt
@@ -1,8 +1,2 @@
 # Python packages to install within the Docker image associated with this Action.
 roboto
-#bokeh==3.0.0
-pyulog
-scipy
-pyfftw
-img2pdf
-selenium
diff --git a/actions/px4_flight_review/scripts/test.sh b/actions/px4_flight_review/scripts/test.sh
@@ -77,11 +77,6 @@ main() {
     run_docker_test ""
     file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.html
 
-    # Test 2
-     echo "Running Test 2: Test .pdf report generation"
-     clean_actual_output
-     run_docker_test "-e ROBOTO_PARAM_SAVE_PDF=True"
-     file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.pdf
 }
 
 # Run the main test execution

diff --git a/actions/px4_flight_review/src/px4_flight_review/__main__.py b/actions/px4_flight_review/src/px4_flight_review/__main__.py
@@ -1,109 +1,90 @@
 import argparse
 import os
+import io
 import pathlib
-from bokeh.plotting import output_file, save
+from jinja2 import Template
 from helper import load_ulog_file
 from pyulog.px4 import PX4ULog
 from db_entry import DBData
 from configured_plots import generate_plots
-import re
-from bokeh.io import export_png
-import img2pdf
+from plotted_tables import get_heading_html
+from bokeh.layouts import column
+from bokeh.resources import CDN
+from bokeh.embed import components
+from bokeh.io import curdoc
 from roboto.domain import actions
 
-
-def extract_plot_html(file_path):
-    """Extracts the HTML content of a Bokeh plot from a saved file."""
-    with open(file_path, 'r') as file:
-        content = file.read()
-        plot_html = re.search(r'(?<=<body>).*(?=</body>)', content, re.DOTALL).group()
-    return plot_html
-
-
-def extract_head_html(file_path):
-    """Extracts the HTML content for Bokeh resources from the <head> section."""
-    with open(file_path, 'r') as file:
-        content = file.read()
-        head_html = re.search(r'(?<=<head>).*(?=</head>)', content, re.DOTALL).group()
-    return head_html
-
-
-def process_ulg_file(ulog_file_path, output_folder, save_pdf):
+def process_ulg_file(ulog_file_path, output_folder):
     """Process a single .ulg file to generate and combine Bokeh plots into an HTML report."""
     ulog = load_ulog_file(ulog_file_path)
     px4_ulog = PX4ULog(ulog)
     db_data = DBData()
-    vehicle_data = None  # You will need to determine how to set this
-
-    log_id = "av"
-    link_to_3d_page = '3d?log=' + log_id
-    link_to_pid_analysis_page = '?plots=pid_analysis&log=' + log_id
-
-    plots = generate_plots(ulog, px4_ulog, db_data, vehicle_data, link_to_3d_page, link_to_pid_analysis_page)
+    vehicle_data = None
+
+    plots = generate_plots(
+        ulog,
+        px4_ulog,
+        db_data,
+        vehicle_data,
+        None,
+        "",
+    )
 
     # Ensure the output directory exists
     output_folder.mkdir(parents=True, exist_ok=True)
-    report_filename = pathlib.Path(ulog_file_path).stem + '_report.html'
+    report_filename = pathlib.Path(ulog_file_path).stem + "_report.html"
     report_filepath = output_folder / report_filename
 
-    # Save individual plots as PNG and create HTML content
-    combined_html_content = ""
-    png_files = []  # List to store paths of PNG files
-
-    for i, plot in enumerate(plots):
-        try:
-            if save_pdf:
-            # Export plot as PNG
-                png_filename = report_filepath.with_suffix(f'.{i}.png')
-                export_png(plot, filename=png_filename)
-                png_files.append(png_filename)
-
-            # Create HTML content (optional if you only need the PDF)
-            plot_file = report_filepath.with_suffix(f'.{i}.html')
-            output_file(plot_file)
-            save(plot)
-
-            if i == 0:  # Extract head from first plot
-                combined_html_content += f"<html><head>{extract_head_html(plot_file)}</head><body>"
-            combined_html_content += extract_plot_html(plot_file)
-
-            # Delete the individual plot file
-            plot_file.unlink(missing_ok=True)
-        except Exception as e:
-            print(f"Error processing plot {i}: {e}")
-
-    combined_html_content += "</body></html>"
-
-    # Save as HTML (optional if you only need the PDF)
-    with open(report_filepath, 'w') as file:
-        file.write(combined_html_content)
-    print(f"Report generated: {report_filepath}")
-
-    if save_pdf:
-        # Combine PNG files into a single PDF
-        pdf_filename = report_filepath.with_suffix('.pdf')
-        with open(pdf_filename, "wb") as f:
-            f.write(img2pdf.convert(png_files))
-        print(f"PDF report generated: {pdf_filename}")
-
-        # Optional: Cleanup PNG files after PDF creation
-        for png_file in png_files:
-            png_file.unlink()
+    # Load Jinja HTML template for summary and plots
+    dirname = os.path.dirname(__file__)
+    j2_template = os.path.join(dirname, 'index.html')
+
+    with open(j2_template) as f:
+        template = Template(f.read())
+
+    # Arrange all generated plots into a column
+    layout = column(plots)
+
+    # Extract Bokeh HTML components so we can lay out our own file
+    # instead of relying on Bokeh's entirely self-generated HTML
+    script_bokeh, div_bokeh = components(layout)
+    resources_bokeh = CDN.render()
+
+    # Compose HTML file with bokeh plots and additional summary data
+    html = template.render(
+        resources=resources_bokeh,
+        script=script_bokeh,
+        plots=div_bokeh,
+        title_html=get_heading_html(ulog, px4_ulog, db_data, None),
+        hardfault_html=curdoc().template_variables.get("hardfault_html"),
+        corrupt_log_html=curdoc().template_variables.get("corrupt_log_html"),
+        error_labels_html=curdoc().template_variables.get("error_labels_html"),
+        info_table_html=curdoc().template_variables.get("info_table_html"),
+        has_position_data=curdoc().template_variables.get("has_position_data"),
+        mapbox_api_access_token=os.environ.get("ROBOTO_PARAM_MAPBOX_API_TOKEN"),
+        pos_datas=curdoc().template_variables.get("pos_datas"),
+        pos_flight_modes=curdoc().template_variables.get("pos_flight_modes"),
+    )
 
+    # Save the HTML file
+    with io.open(report_filepath, mode="w") as f:
+        f.write(html)
 
-def process_ulg_files(input_folder, output_folder, save_pdf):
+def process_ulg_files(input_folder, output_folder):
     """Process all .ulg files within a given folder and its subdirectories."""
     for root, dirs, files in os.walk(input_folder):
         for file in files:
             if file.endswith(".ulg"):
                 full_path = os.path.join(root, file)
                 relative_path = pathlib.Path(root).relative_to(input_folder)
                 output_path = output_folder / relative_path
-                process_ulg_file(full_path, output_path, save_pdf)
+                process_ulg_file(full_path, output_path)
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Generate Bokeh plot reports from ULG files.')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Generate Bokeh plot reports from ULG files."
+    )
     parser.add_argument(
         "-i",
         "--input-dir",
@@ -123,15 +104,6 @@ def process_ulg_files(input_folder, output_folder, save_pdf):
         default=os.environ.get(actions.InvocationEnvVar.OutputDir.value),
     )
 
-    parser.add_argument(
-        "--save-pdf",
-        action="store_true",
-        required=False,
-        help="Set True to save PDF reports",
-        default=(os.environ.get("ROBOTO_PARAM_SAVE_PDF") == "True"),
-    )
-
     args = parser.parse_args()
 
-    process_ulg_files(args.input_dir, args.output_dir, args.save_pdf)
-
+    process_ulg_files(args.input_dir, args.output_dir)