From fac02534bab368206e5306f9a85fc377148951c6 Mon Sep 17 00:00:00 2001 From: Benji Barash Date: Mon, 22 Jan 2024 20:43:20 -0800 Subject: [PATCH] Improved generation of HTML file in px4_flight_review action Context: The px4_flight_review action generates an HTML file using the underlying PX4/flight_review code. This code is able to generate an HTML file including a flight map, Bokeh plots and log tables. However, the current action implementation generated an HTML file that only included the Bokeh plots. Solution: Refactor how the action generates an HTML file, by better utilizing functions available in Bokeh as well as a Jinja template to lay out the resulting page with map and tables included. The map required inclusion of leaflet and mapbox script dependencies as well as an API token that is now exposed as an action parameter so it can be overridden by end users. Testing: Verified behavior by running the ./build.sh and ./test.sh scripts. Deployed action to Roboto dev stack account and ran with a variety of PX4 ULog files. Observed improved HTML artifacts that are much closer to the public PX4 flight review equivalents. --- .gitignore | 3 + actions/px4_flight_review/Dockerfile | 19 -- actions/px4_flight_review/action.json | 6 +- .../requirements.runtime.txt | 6 - actions/px4_flight_review/scripts/test.sh | 5 - .../src/px4_flight_review/__main__.py | 146 ++++++-------- .../src/px4_flight_review/index.html | 188 ++++++++++++++++++ 7 files changed, 253 insertions(+), 120 deletions(-) create mode 100644 actions/px4_flight_review/src/px4_flight_review/index.html diff --git a/.gitignore b/.gitignore index 68bc17f..2b61a14 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,6 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +# OS X +.DS_Store diff --git a/actions/px4_flight_review/Dockerfile b/actions/px4_flight_review/Dockerfile index 39b58e6..b2a5294 100644 --- a/actions/px4_flight_review/Dockerfile +++ b/actions/px4_flight_review/Dockerfile @@ -8,7 +8,6 @@ ENV DEBIAN_FRONTEND noninteractive ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 - # Install dependencies required for Python 3.10 RUN apt-get update && \ apt-get install -y software-properties-common && \ @@ -26,24 +25,6 @@ RUN apt-get update && apt-get install -y \ xvfb \ && rm -rf /var/lib/apt/lists/* -# Define the Firefox and geckodriver versions -ENV FIREFOX_VERSION 92.0 -ENV GECKODRIVER_VERSION 0.31.0 - -# Install Firefox -RUN wget --no-verbose -O /tmp/firefox.tar.bz2 \ - https://ftp.mozilla.org/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 \ - && tar -xjf /tmp/firefox.tar.bz2 -C /opt/ \ - && ln -s /opt/firefox/firefox /usr/local/bin/firefox \ - && rm /tmp/firefox.tar.bz2 - -# Install geckodriver -RUN wget --no-verbose -O /tmp/geckodriver.tar.gz \ - https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-linux64.tar.gz \ - && tar -xzf /tmp/geckodriver.tar.gz -C /opt/ \ - && ln -s /opt/geckodriver /usr/local/bin/geckodriver \ - && rm /tmp/geckodriver.tar.gz - # Upgrade pip RUN python3.10 -m pip install --upgrade pip diff --git a/actions/px4_flight_review/action.json b/actions/px4_flight_review/action.json index de977ed..7d05ace 100644 --- a/actions/px4_flight_review/action.json +++ b/actions/px4_flight_review/action.json @@ -9,10 +9,10 @@ }, "parameters": [ { - "name": "SAVE_PDF", + "name": "MAPBOX_API_TOKEN", "required": false, - "description": "Set True to save a .pdf", - "default": "False" + "description": "Mapbox API Access Token", + "default": "pk.eyJ1IjoiYmt1ZW5nIiwiYSI6ImNqb3p2Mjl3ZjAwbDAzdm51YTIxdTBra3kifQ.aEC7iOQQYMshIzS_vTxtlA" } ], "tags": ["px4"], diff --git a/actions/px4_flight_review/requirements.runtime.txt b/actions/px4_flight_review/requirements.runtime.txt index f1e00e2..f24f39a 100644 --- a/actions/px4_flight_review/requirements.runtime.txt +++ b/actions/px4_flight_review/requirements.runtime.txt @@ -1,8 +1,2 @@ # Python packages to install within the Docker image associated with this Action. roboto -#bokeh==3.0.0 -pyulog -scipy -pyfftw -img2pdf -selenium diff --git a/actions/px4_flight_review/scripts/test.sh b/actions/px4_flight_review/scripts/test.sh index be8d072..20cb5ed 100755 --- a/actions/px4_flight_review/scripts/test.sh +++ b/actions/px4_flight_review/scripts/test.sh @@ -77,11 +77,6 @@ main() { run_docker_test "" file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.html - # Test 2 - echo "Running Test 2: Test .pdf report generation" - clean_actual_output - run_docker_test "-e ROBOTO_PARAM_SAVE_PDF=True" - file_exists_or_error $ACTUAL_OUTPUT_DIR/test_report.pdf } # Run the main test execution diff --git a/actions/px4_flight_review/src/px4_flight_review/__main__.py b/actions/px4_flight_review/src/px4_flight_review/__main__.py index 4158e79..933ac47 100644 --- a/actions/px4_flight_review/src/px4_flight_review/__main__.py +++ b/actions/px4_flight_review/src/px4_flight_review/__main__.py @@ -1,97 +1,76 @@ import argparse import os +import io import pathlib -from bokeh.plotting import output_file, save +from jinja2 import Template from helper import load_ulog_file from pyulog.px4 import PX4ULog from db_entry import DBData from configured_plots import generate_plots -import re -from bokeh.io import export_png -import img2pdf +from plotted_tables import get_heading_html +from bokeh.layouts import column +from bokeh.resources import CDN +from bokeh.embed import components +from bokeh.io import curdoc from roboto.domain import actions - -def extract_plot_html(file_path): - """Extracts the HTML content of a Bokeh plot from a saved file.""" - with open(file_path, 'r') as file: - content = file.read() - plot_html = re.search(r'(?<=).*(?=)', content, re.DOTALL).group() - return plot_html - - -def extract_head_html(file_path): - """Extracts the HTML content for Bokeh resources from the section.""" - with open(file_path, 'r') as file: - content = file.read() - head_html = re.search(r'(?<=).*(?=)', content, re.DOTALL).group() - return head_html - - -def process_ulg_file(ulog_file_path, output_folder, save_pdf): +def process_ulg_file(ulog_file_path, output_folder): """Process a single .ulg file to generate and combine Bokeh plots into an HTML report.""" ulog = load_ulog_file(ulog_file_path) px4_ulog = PX4ULog(ulog) db_data = DBData() - vehicle_data = None # You will need to determine how to set this - - log_id = "av" - link_to_3d_page = '3d?log=' + log_id - link_to_pid_analysis_page = '?plots=pid_analysis&log=' + log_id - - plots = generate_plots(ulog, px4_ulog, db_data, vehicle_data, link_to_3d_page, link_to_pid_analysis_page) + vehicle_data = None + + plots = generate_plots( + ulog, + px4_ulog, + db_data, + vehicle_data, + None, + "", + ) # Ensure the output directory exists output_folder.mkdir(parents=True, exist_ok=True) - report_filename = pathlib.Path(ulog_file_path).stem + '_report.html' + report_filename = pathlib.Path(ulog_file_path).stem + "_report.html" report_filepath = output_folder / report_filename - # Save individual plots as PNG and create HTML content - combined_html_content = "" - png_files = [] # List to store paths of PNG files - - for i, plot in enumerate(plots): - try: - if save_pdf: - # Export plot as PNG - png_filename = report_filepath.with_suffix(f'.{i}.png') - export_png(plot, filename=png_filename) - png_files.append(png_filename) - - # Create HTML content (optional if you only need the PDF) - plot_file = report_filepath.with_suffix(f'.{i}.html') - output_file(plot_file) - save(plot) - - if i == 0: # Extract head from first plot - combined_html_content += f"{extract_head_html(plot_file)}" - combined_html_content += extract_plot_html(plot_file) - - # Delete the individual plot file - plot_file.unlink(missing_ok=True) - except Exception as e: - print(f"Error processing plot {i}: {e}") - - combined_html_content += "" - - # Save as HTML (optional if you only need the PDF) - with open(report_filepath, 'w') as file: - file.write(combined_html_content) - print(f"Report generated: {report_filepath}") - - if save_pdf: - # Combine PNG files into a single PDF - pdf_filename = report_filepath.with_suffix('.pdf') - with open(pdf_filename, "wb") as f: - f.write(img2pdf.convert(png_files)) - print(f"PDF report generated: {pdf_filename}") - - # Optional: Cleanup PNG files after PDF creation - for png_file in png_files: - png_file.unlink() + # Load Jinja HTML template for summary and plots + dirname = os.path.dirname(__file__) + j2_template = os.path.join(dirname, 'index.html') + + with open(j2_template) as f: + template = Template(f.read()) + + # Arrange all generated plots into a column + layout = column(plots) + + # Extract Bokeh HTML components so we can lay out our own file + # instead of relying on Bokeh's entirely self-generated HTML + script_bokeh, div_bokeh = components(layout) + resources_bokeh = CDN.render() + + # Compose HTML file with bokeh plots and additional summary data + html = template.render( + resources=resources_bokeh, + script=script_bokeh, + plots=div_bokeh, + title_html=get_heading_html(ulog, px4_ulog, db_data, None), + hardfault_html=curdoc().template_variables.get("hardfault_html"), + corrupt_log_html=curdoc().template_variables.get("corrupt_log_html"), + error_labels_html=curdoc().template_variables.get("error_labels_html"), + info_table_html=curdoc().template_variables.get("info_table_html"), + has_position_data=curdoc().template_variables.get("has_position_data"), + mapbox_api_access_token=os.environ.get("ROBOTO_PARAM_MAPBOX_API_TOKEN"), + pos_datas=curdoc().template_variables.get("pos_datas"), + pos_flight_modes=curdoc().template_variables.get("pos_flight_modes"), + ) + # Save the HTML file + with io.open(report_filepath, mode="w") as f: + f.write(html) -def process_ulg_files(input_folder, output_folder, save_pdf): +def process_ulg_files(input_folder, output_folder): """Process all .ulg files within a given folder and its subdirectories.""" for root, dirs, files in os.walk(input_folder): for file in files: @@ -99,11 +78,13 @@ def process_ulg_files(input_folder, output_folder, save_pdf): full_path = os.path.join(root, file) relative_path = pathlib.Path(root).relative_to(input_folder) output_path = output_folder / relative_path - process_ulg_file(full_path, output_path, save_pdf) + process_ulg_file(full_path, output_path) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Generate Bokeh plot reports from ULG files.') +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate Bokeh plot reports from ULG files." + ) parser.add_argument( "-i", "--input-dir", @@ -123,15 +104,6 @@ def process_ulg_files(input_folder, output_folder, save_pdf): default=os.environ.get(actions.InvocationEnvVar.OutputDir.value), ) - parser.add_argument( - "--save-pdf", - action="store_true", - required=False, - help="Set True to save PDF reports", - default=(os.environ.get("ROBOTO_PARAM_SAVE_PDF") == "True"), - ) - args = parser.parse_args() - process_ulg_files(args.input_dir, args.output_dir, args.save_pdf) - + process_ulg_files(args.input_dir, args.output_dir) diff --git a/actions/px4_flight_review/src/px4_flight_review/index.html b/actions/px4_flight_review/src/px4_flight_review/index.html new file mode 100644 index 0000000..ec3031d --- /dev/null +++ b/actions/px4_flight_review/src/px4_flight_review/index.html @@ -0,0 +1,188 @@ + + + + + Flight Review + {{ resources }} + {{ script }} + + + + + + + + +
+
+ Do you need help with interpreting the plots? See + here. +
+ +
+ {{ title_html }} + + {% if hardfault_html != None %} + {{hardfault_html }} + {% endif %} + + {{ info_table_html }} + + {% if corrupt_log_html != None %} + {{ corrupt_log_html }} + {% endif %} + + {% if error_labels_html != None %} + {{ error_labels_html }} + {% endif %} + +
+ + {% if has_position_data %} + +
+ + + {% endif %} + +
{{ plots }}
+
+ +