diff --git a/clean_for_build.py b/clean_for_build.py deleted file mode 100644 index 7cee04975..000000000 --- a/clean_for_build.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import os.path -import shutil - - -def clean(): - """ - Delete directories which are created during sdist / wheel build process. - - Cross-platform method as an alternative to juggling between: - Linux/mac: rm -rf [dirs] - Windows: rm -Recurse -Force [dirs] - """ - here = os.path.dirname(__file__) - dirs = ["build", "dist", "pyxform.egg-info"] - for d in dirs: - path = os.path.join(here, d) - if os.path.exists(path): - print("Removing:", path) - shutil.rmtree(path) - - -if __name__ == "__main__": - clean() diff --git a/pyxform/constants.py b/pyxform/constants.py index 6bd41cc6a..74adefbe6 100644 --- a/pyxform/constants.py +++ b/pyxform/constants.py @@ -34,6 +34,7 @@ SUBMISSION_URL = "submission_url" AUTO_SEND = "auto_send" AUTO_DELETE = "auto_delete" +DEFAULT_FORM_NAME = "data" DEFAULT_LANGUAGE_KEY = "default_language" DEFAULT_LANGUAGE_VALUE = "default" LABEL = "label" diff --git a/pyxform/entities/entities_parsing.py b/pyxform/entities/entities_parsing.py index 348537972..51f21bc67 100644 --- a/pyxform/entities/entities_parsing.py +++ b/pyxform/entities/entities_parsing.py @@ -72,7 +72,7 @@ def get_validated_dataset_name(entity): if not is_valid_xml_tag(dataset): if isinstance(dataset, bytes): - dataset = dataset.encode("utf-8") + dataset = dataset.decode("utf-8") raise PyXFormError( f"Invalid entity list name: '{dataset}'. Names must begin with a letter, colon, or underscore. Other characters can include numbers or dashes." @@ -117,7 +117,7 @@ def validate_entity_saveto( if not is_valid_xml_tag(save_to): if isinstance(save_to, bytes): - save_to = save_to.encode("utf-8") + save_to = save_to.decode("utf-8") raise PyXFormError( f"{error_start} '{save_to}'. Entity property names {const.XML_IDENTIFIER_ERROR_MESSAGE}" diff --git a/pyxform/errors.py b/pyxform/errors.py index 51aaf3845..89ca6ff96 100644 --- a/pyxform/errors.py +++ b/pyxform/errors.py @@ -9,3 +9,7 @@ class PyXFormError(Exception): class ValidationError(PyXFormError): """Common base class for pyxform validation exceptions.""" + + +class PyXFormReadError(PyXFormError): + """Common base class for pyxform exceptions occuring during reading XLSForm data.""" diff --git a/pyxform/instance.py b/pyxform/instance.py index fb427af26..17b77f9f7 100644 --- a/pyxform/instance.py +++ b/pyxform/instance.py @@ -2,6 +2,8 @@ SurveyInstance class module. """ +import os.path + from pyxform.errors import PyXFormError from pyxform.xform_instance_parser import parse_xform_instance @@ -76,8 +78,6 @@ def answers(self): return self._answers def import_from_xml(self, xml_string_or_filename): - import os.path - if os.path.isfile(xml_string_or_filename): xml_str = open(xml_string_or_filename, encoding="utf-8").read() else: diff --git a/pyxform/survey.py b/pyxform/survey.py index 3a2b8e158..d5337b60a 100644 --- a/pyxform/survey.py +++ b/pyxform/survey.py @@ -10,6 +10,7 @@ from collections.abc import Generator, Iterator from datetime import datetime from functools import lru_cache +from pathlib import Path from pyxform import aliases, constants from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS, NSMAP @@ -970,10 +971,10 @@ def date_stamp(self): """Returns a date string with the format of %Y_%m_%d.""" return self._created.strftime("%Y_%m_%d") - def _to_ugly_xml(self): + def _to_ugly_xml(self) -> str: return '' + self.xml().toxml() - def _to_pretty_xml(self): + def _to_pretty_xml(self) -> str: """Get the XForm with human readable formatting.""" return '\n' + self.xml().toprettyxml(indent=" ") @@ -1171,10 +1172,9 @@ def _var_repl_output_function(matchobj): else: return text, False - # pylint: disable=too-many-arguments def print_xform_to_file( self, path=None, validate=True, pretty_print=True, warnings=None, enketo=False - ): + ) -> str: """ Print the xForm to a file and optionally validate it as well by throwing exceptions and adding warnings to the warnings array. @@ -1183,12 +1183,13 @@ def print_xform_to_file( warnings = [] if not path: path = self._print_name + ".xml" + if pretty_print: + xml = self._to_pretty_xml() + else: + xml = self._to_ugly_xml() try: with open(path, mode="w", encoding="utf-8") as file_obj: - if pretty_print: - file_obj.write(self._to_pretty_xml()) - else: - file_obj.write(self._to_ugly_xml()) + file_obj.write(xml) except Exception: if os.path.exists(path): os.unlink(path) @@ -1210,6 +1211,7 @@ def print_xform_to_file( + ". " + "Learn more: http://xlsform.org#multiple-language-support" ) + return xml def to_xml(self, validate=True, pretty_print=True, warnings=None, enketo=False): """ @@ -1227,7 +1229,7 @@ def to_xml(self, validate=True, pretty_print=True, warnings=None, enketo=False): tmp.close() try: # this will throw an exception if the xml is not valid - self.print_xform_to_file( + xml = self.print_xform_to_file( path=tmp.name, validate=validate, pretty_print=pretty_print, @@ -1235,12 +1237,8 @@ def to_xml(self, validate=True, pretty_print=True, warnings=None, enketo=False): enketo=enketo, ) finally: - if os.path.exists(tmp.name): - os.remove(tmp.name) - if pretty_print: - return self._to_pretty_xml() - - return self._to_ugly_xml() + Path(tmp.name).unlink(missing_ok=True) + return xml def instantiate(self): """ diff --git a/pyxform/utils.py b/pyxform/utils.py index a29b2d6cb..5e362e8da 100644 --- a/pyxform/utils.py +++ b/pyxform/utils.py @@ -7,17 +7,16 @@ import json import os import re +from io import StringIO from json.decoder import JSONDecodeError -from typing import NamedTuple +from typing import Any, NamedTuple from xml.dom import Node from xml.dom.minidom import Element, Text, _write_data -import openpyxl -import xlrd from defusedxml.minidom import parseString +from pyxform import constants as const from pyxform.errors import PyXFormError -from pyxform.xls2json_backends import is_empty, xls_value_to_unicode, xlsx_value_to_str SEP = "_" @@ -167,66 +166,32 @@ def flatten(li): yield from subli -def sheet_to_csv(workbook_path, csv_path, sheet_name): - if workbook_path.endswith(".xls"): - return xls_sheet_to_csv(workbook_path, csv_path, sheet_name) - else: - return xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name) - +def external_choices_to_csv( + workbook_dict: dict[str, Any], warnings: list | None = None +) -> str | None: + """ + Convert the 'external_choices' sheet data to CSV. -def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): - wb = xlrd.open_workbook(workbook_path) - try: - sheet = wb.sheet_by_name(sheet_name) - except xlrd.biffh.XLRDError: - return False - if not sheet or sheet.nrows < 2: - return False - with open(csv_path, mode="w", encoding="utf-8", newline="") as f: - writer = csv.writer(f, quoting=csv.QUOTE_ALL) - mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] - for row_idx in range(sheet.nrows): - csv_data = [] - try: - for v, m in zip(sheet.row(row_idx), mask, strict=False): - if m: - value = v.value - value_type = v.ctype - data = xls_value_to_unicode(value, value_type, wb.datemode) - # clean the values of leading and trailing whitespaces - data = data.strip() - csv_data.append(data) - except TypeError: - continue - writer.writerow(csv_data) - - return True - - -def xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name): - wb = openpyxl.open(workbook_path, read_only=True, data_only=True) + :param workbook_dict: The result from xls2json.workbook_to_json. + :param warnings: The conversions warnings list. + """ + warnings = coalesce(warnings, []) + if const.EXTERNAL_CHOICES not in workbook_dict: + warnings.append( + f"Could not export itemsets.csv, the '{const.EXTERNAL_CHOICES}' sheet is missing." + ) + return None + + itemsets = StringIO(newline="") + csv_writer = csv.writer(itemsets, quoting=csv.QUOTE_ALL) try: - sheet = wb[sheet_name] - except KeyError: - return False - - with open(csv_path, mode="w", encoding="utf-8", newline="") as f: - writer = csv.writer(f, quoting=csv.QUOTE_ALL) - mask = [not is_empty(cell.value) for cell in sheet[1]] - for row in sheet.rows: - csv_data = [] - try: - for v, m in zip(row, mask, strict=False): - if m: - data = xlsx_value_to_str(v.value) - # clean the values of leading and trailing whitespaces - data = data.strip() - csv_data.append(data) - except TypeError: - continue - writer.writerow(csv_data) - wb.close() - return True + header = workbook_dict["external_choices_header"][0] + except (IndexError, KeyError, TypeError): + header = {k for d in workbook_dict[const.EXTERNAL_CHOICES] for k in d} + csv_writer.writerow(header) + for row in workbook_dict[const.EXTERNAL_CHOICES]: + csv_writer.writerow(row.values()) + return itemsets.getvalue() def has_external_choices(json_struct): @@ -235,7 +200,11 @@ def has_external_choices(json_struct): """ if isinstance(json_struct, dict): for k, v in json_struct.items(): - if k == "type" and isinstance(v, str) and v.startswith("select one external"): + if ( + k == const.TYPE + and isinstance(v, str) + and v.startswith(const.SELECT_ONE_EXTERNAL) + ): return True elif has_external_choices(v): return True diff --git a/pyxform/xls2json.py b/pyxform/xls2json.py index ad25baa63..1bce494d7 100644 --- a/pyxform/xls2json.py +++ b/pyxform/xls2json.py @@ -22,7 +22,7 @@ ) from pyxform.errors import PyXFormError from pyxform.parsing.expression import is_single_token_expression -from pyxform.utils import PYXFORM_REFERENCE_REGEX, default_is_dynamic +from pyxform.utils import PYXFORM_REFERENCE_REGEX, coalesce, default_is_dynamic from pyxform.validators.pyxform import parameters_generic, select_from_file from pyxform.validators.pyxform.android_package_name import validate_android_package_name from pyxform.validators.pyxform.translations_checks import SheetTranslations @@ -395,7 +395,7 @@ def workbook_to_json( workbook_dict, form_name: str | None = None, fallback_form_name: str | None = None, - default_language: str = constants.DEFAULT_LANGUAGE_VALUE, + default_language: str | None = None, warnings: list[str] | None = None, ) -> dict[str, Any]: """ @@ -416,8 +416,7 @@ def workbook_to_json( returns a nested dictionary equivalent to the format specified in the json form spec. """ - if warnings is None: - warnings = [] + warnings = coalesce(warnings, []) is_valid = False # Sheet names should be case-insensitive workbook_dict = {x.lower(): y for x, y in workbook_dict.items()} @@ -441,8 +440,8 @@ def workbook_to_json( ) # Make sure the passed in vars are unicode - form_name = str(form_name) - default_language = str(default_language) + form_name = str(coalesce(form_name, constants.DEFAULT_FORM_NAME)) + default_language = str(coalesce(default_language, constants.DEFAULT_LANGUAGE_VALUE)) # We check for double columns to determine whether to use them # or single colons to delimit grouped headers. @@ -500,7 +499,9 @@ def workbook_to_json( ) # Here we create our json dict root with default settings: - id_string = settings.get(constants.ID_STRING, fallback_form_name) + id_string = settings.get( + constants.ID_STRING, coalesce(fallback_form_name, constants.DEFAULT_FORM_NAME) + ) sms_keyword = settings.get(constants.SMS_KEYWORD, id_string) json_dict = { constants.TYPE: constants.SURVEY, @@ -970,7 +971,7 @@ def workbook_to_json( question_name = str(row[constants.NAME]) if not is_valid_xml_tag(question_name): if isinstance(question_name, bytes): - question_name = question_name.encode("utf-8") + question_name = question_name.decode("utf-8") raise PyXFormError( f"{ROW_FORMAT_STRING % row_number} Invalid question name '{question_name}'. Names {XML_IDENTIFIER_ERROR_MESSAGE}" @@ -1591,7 +1592,7 @@ def get_filename(path): def parse_file_to_json( path: str, - default_name: str = "data", + default_name: str = constants.DEFAULT_FORM_NAME, default_language: str = constants.DEFAULT_LANGUAGE_VALUE, warnings: list[str] | None = None, file_object: IO | None = None, diff --git a/pyxform/xls2json_backends.py b/pyxform/xls2json_backends.py index 9a5b10d40..6a81a8117 100644 --- a/pyxform/xls2json_backends.py +++ b/pyxform/xls2json_backends.py @@ -4,16 +4,19 @@ import csv import datetime -import os import re from collections import OrderedDict from collections.abc import Callable, Iterator -from contextlib import closing +from dataclasses import dataclass +from enum import Enum from functools import reduce -from io import StringIO +from io import BytesIO, IOBase, StringIO +from os import PathLike +from pathlib import Path from typing import Any from zipfile import BadZipFile +import openpyxl import xlrd from openpyxl.cell import Cell as pyxlCell from openpyxl.reader.excel import ExcelReader @@ -25,7 +28,7 @@ from xlrd.xldate import XLDateAmbiguous from pyxform import constants -from pyxform.errors import PyXFormError +from pyxform.errors import PyXFormError, PyXFormReadError aCell = xlrdCell | pyxlCell XL_DATE_AMBIGOUS_MSG = ( @@ -186,18 +189,14 @@ def process_workbook(wb: xlrdBook): return result_book try: - if isinstance(path_or_file, str | bytes | os.PathLike): - file = open(path_or_file, mode="rb") - else: - file = path_or_file - with closing(file) as wb_file: - workbook = xlrd.open_workbook(file_contents=wb_file.read()) - try: - return process_workbook(wb=workbook) - finally: - workbook.release_resources() - except xlrd.XLRDError as read_err: - raise PyXFormError(f"Error reading .xls file: {read_err}") from read_err + wb_file = get_definition_data(definition=path_or_file) + workbook = xlrd.open_workbook(file_contents=wb_file.data.getvalue()) + try: + return process_workbook(wb=workbook) + finally: + workbook.release_resources() + except (AttributeError, TypeError, xlrd.XLRDError) as read_err: + raise PyXFormReadError(f"Error reading .xls file: {read_err}") from read_err def xls_value_to_unicode(value, value_type, datemode) -> str: @@ -281,20 +280,16 @@ def process_workbook(wb: pyxlWorkbook): return result_book try: - if isinstance(path_or_file, str | bytes | os.PathLike): - file = open(path_or_file, mode="rb") - else: - file = path_or_file - with closing(file) as wb_file: - reader = ExcelReader(wb_file, read_only=True, data_only=True) - reader.read() - try: - return process_workbook(wb=reader.wb) - finally: - reader.wb.close() - reader.archive.close() - except (OSError, BadZipFile, KeyError) as read_err: - raise PyXFormError(f"Error reading .xlsx file: {read_err}") from read_err + wb_file = get_definition_data(definition=path_or_file) + reader = ExcelReader(wb_file.data, read_only=True, data_only=True) + reader.read() + try: + return process_workbook(wb=reader.wb) + finally: + reader.wb.close() + reader.archive.close() + except (BadZipFile, KeyError, OSError, TypeError) as read_err: + raise PyXFormReadError(f"Error reading .xlsx file: {read_err}") from read_err def xlsx_value_to_str(value) -> str: @@ -360,13 +355,6 @@ def replace_prefix(d, prefix): def csv_to_dict(path_or_file): - if isinstance(path_or_file, str): - csv_data = open(path_or_file, encoding="utf-8", newline="") - else: - csv_data = path_or_file - - _dict = OrderedDict() - def first_column_as_sheet_name(row): if len(row) == 0: return None, None @@ -383,31 +371,41 @@ def first_column_as_sheet_name(row): content = None return s_or_c, content - reader = csv.reader(csv_data) - sheet_name = None - current_headers = None - for row in reader: - survey_or_choices, content = first_column_as_sheet_name(row) - if survey_or_choices is not None: - sheet_name = survey_or_choices - if sheet_name not in _dict: - _dict[str(sheet_name)] = [] - current_headers = None - if content is not None: - if current_headers is None: - current_headers = content - _dict[f"{sheet_name}_header"] = _list_to_dict_list(current_headers) - else: - _d = OrderedDict() - for key, val in zip(current_headers, content, strict=False): - if val != "": - # Slight modification so values are striped - # this is because csvs often spaces following commas - # (but the csv reader might already handle that.) - _d[str(key)] = str(val.strip()) - _dict[sheet_name].append(_d) - csv_data.close() - return _dict + def process_csv_data(rd): + _dict = OrderedDict() + sheet_name = None + current_headers = None + for row in rd: + survey_or_choices, content = first_column_as_sheet_name(row) + if survey_or_choices is not None: + sheet_name = survey_or_choices + if sheet_name not in _dict: + _dict[str(sheet_name)] = [] + current_headers = None + if content is not None: + if current_headers is None: + current_headers = content + _dict[f"{sheet_name}_header"] = _list_to_dict_list(current_headers) + else: + _d = OrderedDict() + for key, val in zip(current_headers, content, strict=False): + if val != "": + # Slight modification so values are striped + # this is because csvs often spaces following commas + # (but the csv reader might already handle that.) + _d[str(key)] = str(val.strip()) + _dict[sheet_name].append(_d) + return _dict + + try: + csv_data = get_definition_data(definition=path_or_file) + csv_str = csv_data.data.getvalue().decode("utf-8") + if not is_csv(data=csv_str): + raise PyXFormError("The input data does not appear to be a valid XLSForm.") # noqa: TRY301 + reader = csv.reader(StringIO(initial_value=csv_str, newline="")) + return process_csv_data(rd=reader) + except (AttributeError, PyXFormError) as read_err: + raise PyXFormReadError(f"Error reading .csv file: {read_err}") from read_err """ @@ -460,3 +458,338 @@ def convert_file_to_csv_string(path): for out_row in out_rows: writer.writerow([None, *out_row]) return foo.getvalue() + + +def sheet_to_csv(workbook_path, csv_path, sheet_name): + if workbook_path.endswith(".xls"): + return xls_sheet_to_csv(workbook_path, csv_path, sheet_name) + else: + return xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name) + + +def xls_sheet_to_csv(workbook_path, csv_path, sheet_name): + wb = xlrd.open_workbook(workbook_path) + try: + sheet = wb.sheet_by_name(sheet_name) + except xlrd.biffh.XLRDError: + return False + if not sheet or sheet.nrows < 2: + return False + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + mask = [v and len(v.strip()) > 0 for v in sheet.row_values(0)] + for row_idx in range(sheet.nrows): + csv_data = [] + try: + for v, m in zip(sheet.row(row_idx), mask, strict=False): + if m: + value = v.value + value_type = v.ctype + data = xls_value_to_unicode(value, value_type, wb.datemode) + # clean the values of leading and trailing whitespaces + data = data.strip() + csv_data.append(data) + except TypeError: + continue + writer.writerow(csv_data) + + return True + + +def xlsx_sheet_to_csv(workbook_path, csv_path, sheet_name): + wb = openpyxl.open(workbook_path, read_only=True, data_only=True) + try: + sheet = wb[sheet_name] + except KeyError: + return False + + with open(csv_path, mode="w", encoding="utf-8", newline="") as f: + writer = csv.writer(f, quoting=csv.QUOTE_ALL) + mask = [not is_empty(cell.value) for cell in sheet[1]] + for row in sheet.rows: + csv_data = [] + try: + for v, m in zip(row, mask, strict=False): + if m: + data = xlsx_value_to_str(v.value) + # clean the values of leading and trailing whitespaces + data = data.strip() + csv_data.append(data) + except TypeError: + continue + writer.writerow(csv_data) + wb.close() + return True + + +MD_COMMENT = re.compile(r"^\s*#") +MD_COMMENT_INLINE = re.compile(r"^(.*)(#[^|]+)$") +MD_CELL = re.compile(r"\s*\|(.*)\|\s*") +MD_SEPARATOR = re.compile(r"^[\|-]+$") +MD_PIPE_OR_ESCAPE = re.compile(r"(? list[tuple[str, list[list[str]]]]: + ss_arr = [] + for item in mdstr.split("\n"): + arr = _md_extract_array(item) + if arr: + ss_arr.append(arr) + sheet_name = False + sheet_arr = False + sheets = [] + for row in ss_arr: + if row[0] is not None: + if sheet_arr: + sheets.append((sheet_name, sheet_arr)) + sheet_arr = [] + sheet_name = row[0] + excluding_first_col = row[1:] + if sheet_name and not _md_is_null_row(excluding_first_col): + sheet_arr.append(excluding_first_col) + sheets.append((sheet_name, sheet_arr)) + + return sheets + + +def md_to_dict(md: str | BytesIO): + def _row_to_dict(row, headers): + out_dict = {} + for i in range(len(row)): + col = row[i] + if col not in [None, ""]: + out_dict[headers[i]] = col + return out_dict + + def list_to_dicts(arr): + return [_row_to_dict(r, arr[0]) for r in arr[1:]] + + def process_md_data(md_: str): + _md = [] + for line in md_.split("\n"): + if re.match(MD_COMMENT, line): + # ignore lines which start with pound sign + continue + elif re.match(MD_COMMENT_INLINE, line): + # keep everything before the # outside of the last occurrence of | + _md.append(re.match(MD_COMMENT_INLINE, line).groups()[0].strip()) + else: + _md.append(line.strip()) + md_ = "\n".join(_md) + sheets = {} + for sheet, contents in _md_table_to_ss_structure(md_): + sheets[sheet] = list_to_dicts(contents) + return sheets + + try: + md_data = get_definition_data(definition=md) + md_str = md_data.data.getvalue().decode("utf-8") + if not is_markdown_table(data=md_str): + raise PyXFormError("The input data does not appear to be a valid XLSForm.") # noqa: TRY301 + return process_md_data(md_=md_str) + except (AttributeError, PyXFormError, TypeError) as read_err: + raise PyXFormReadError(f"Error reading .md file: {read_err}") from read_err + + +def md_table_to_workbook(mdstr: str) -> pyxlWorkbook: + """ + Convert Markdown table string to an openpyxl.Workbook. Call wb.save() to persist. + """ + md_data = _md_table_to_ss_structure(mdstr=mdstr) + wb = pyxlWorkbook(write_only=True) + for key, rows in md_data: + sheet = wb.create_sheet(title=key) + for r in rows: + sheet.append(r) + return wb + + +def count_characters_limit(data: str, find: str, limit: int) -> int: + count = 0 + for c in data: + if c == find: + count += 1 + if count == limit: + break + return count + + +def is_markdown_table(data: str) -> bool: + """ + Does the string look like a markdown table? Checks the first 5KB. + + A minimal form with one question requires 10 pipe characters: + + | survey | + | | type | name | + | | integer | a | + + Minimum to parse at all is 5 pipes. + + | survey | + | | foo | + + :param data: The data to check. + """ + return 5 <= count_characters_limit(data[:5000], "|", 5) + + +def is_csv(data: str) -> bool: + """ + Does the string look like a CSV? Checks the first 5KB. + + A minimal form with one question requires 4 comma characters: + + "survey" + ,"type","name" + ,"integer","a" + + :param data: The data to check. + """ + return 4 <= count_characters_limit(data[:5000], ",", 4) + + +class SupportedFileTypes(Enum): + xlsx = ".xlsx" + xlsm = ".xlsm" + xls = ".xls" + md = ".md" + csv = ".csv" + + @staticmethod + def get_processors(): + return { + SupportedFileTypes.xlsx: xlsx_to_dict, + SupportedFileTypes.xls: xls_to_dict, + SupportedFileTypes.md: md_to_dict, + SupportedFileTypes.csv: csv_to_dict, + } + + +@dataclass +class Definition: + data: BytesIO + file_type: SupportedFileTypes | None + file_path_stem: str | None + + +def definition_to_dict( + definition: str | PathLike[str] | bytes | BytesIO | IOBase | Definition, + file_type: str | None = None, +) -> dict: + """ + Convert raw definition data to a dict ready for conversion to a XForm. + + :param definition: XLSForm definition data. + :param file_type: If provided, attempt parsing the data only as this type. Otherwise, + parsing of supported data types will be attempted until one of them succeeds. + :return: + """ + supported = f"Must be one of: {', '.join(t.value for t in SupportedFileTypes)}" + processors = SupportedFileTypes.get_processors() + if file_type is not None: + try: + ft = SupportedFileTypes(file_type) + except ValueError as err: + raise PyXFormError( + f"Argument 'file_type' is not a supported type. {supported}" + ) from err + else: + processors = {ft: processors[ft]} + + for func in processors.values(): + try: + return func(definition) + except PyXFormReadError: # noqa: PERF203 + continue + + raise PyXFormError( + f"Argument 'definition' was not recognized as a supported type. {supported}" + ) + + +def get_definition_data( + definition: str | PathLike[str] | bytes | BytesIO | IOBase | Definition, +) -> Definition: + """ + Get the form definition data from a path or bytes. + + :param definition: The path to the file to upload (string or PathLike), or the + form definition in memory (string or bytes). + """ + if isinstance(definition, Definition): + return definition + definition_data = None + file_type = None + file_path_stem = None + + # Read in data from paths, or failing that try to process the string. + if isinstance(definition, str | PathLike): + file_read = False + try: + file_path = Path(definition) + + except TypeError as err: + raise PyXFormError( + "Parameter 'definition' does not appear to be a valid file path." + ) from err + try: + file_exists = file_path.is_file() + except (FileNotFoundError, OSError): + pass + else: + if file_exists: + file_path_stem = file_path.stem + try: + file_type = SupportedFileTypes(file_path.suffix) + except ValueError: + # The suffix was not a useful hint but we can try to parse anyway. + pass + definition = BytesIO(file_path.read_bytes()) + file_read = True + if not file_read and isinstance(definition, str): + definition = definition.encode("utf-8") + + # io.IOBase seems about at close as possible to the hint typing.BinaryIO. + if isinstance(definition, bytes | BytesIO | IOBase): + # Normalise to BytesIO. + if isinstance(definition, bytes): + definition_data = BytesIO(definition) + elif isinstance(definition, BytesIO): # BytesIO is a subtype of IOBase + definition_data = definition + else: + definition_data = BytesIO(definition.read()) + + return Definition( + data=definition_data, + file_type=file_type, + file_path_stem=file_path_stem, + ) diff --git a/pyxform/xls2xform.py b/pyxform/xls2xform.py index d9644b604..f9af81a3e 100644 --- a/pyxform/xls2xform.py +++ b/pyxform/xls2xform.py @@ -6,12 +6,23 @@ import argparse import json import logging -import os +from dataclasses import dataclass +from io import BytesIO +from os import PathLike from os.path import splitext +from pathlib import Path +from typing import TYPE_CHECKING, BinaryIO from pyxform import builder, xls2json -from pyxform.utils import has_external_choices, sheet_to_csv +from pyxform.utils import coalesce, external_choices_to_csv, has_external_choices from pyxform.validators.odk_validate import ODKValidateError +from pyxform.xls2json_backends import ( + definition_to_dict, + get_definition_data, +) + +if TYPE_CHECKING: + from pyxform.survey import Survey logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) @@ -28,35 +39,117 @@ def get_xml_path(path): return splitext(path)[0] + ".xml" -def xls2xform_convert( - xlsform_path, xform_path, validate=True, pretty_print=True, enketo=False -): - warnings = [] +@dataclass +class ConvertResult: + """ + Result data from the XLSForm to XForm conversion. + + :param xform: The result XForm + :param warnings: Warnings raised during conversion. + :param itemsets: If the XLSForm defined external itemsets, a CSV version of them. + :param _pyxform: Internal representation of the XForm, may change without notice. + :param _survey: Internal representation of the XForm, may change without notice. + """ + + xform: str + warnings: list[str] + itemsets: str | None + _pyxform: dict + _survey: "Survey" + - json_survey = xls2json.parse_file_to_json(xlsform_path, warnings=warnings) - survey = builder.create_survey_element_from_dict(json_survey) - # Setting validate to false will cause the form not to be processed by - # ODK Validate. - # This may be desirable since ODK Validate requires launching a subprocess - # that runs some java code. - survey.print_xform_to_file( - xform_path, +def convert( + xlsform: str | PathLike[str] | bytes | BytesIO | BinaryIO | dict, + warnings: list[str] | None = None, + validate: bool = False, + pretty_print: bool = False, + enketo: bool = False, + form_name: str | None = None, + default_language: str | None = None, + file_type: str | None = None, +) -> ConvertResult: + """ + Run the XLSForm to XForm conversion. + + This function avoids result file IO so it is more suited to library usage of pyxform. + + If validate=True or Enketo=True, then the XForm will be written to a temporary file + to be checked by ODK Validate and/or Enketo Validate. These validators are run as + external processes. A recent version of ODK Validate is distributed with pyxform, + while Enketo Validate is not. A script to download or update these validators is + provided in `validators/updater.py`. + + :param xlsform: The input XLSForm file path or content. If the content is bytes or + supports read (a class that has read() -> bytes) it's assumed to relate to the file + bytes content, not a path. + :param warnings: The conversions warnings list. + :param validate: If True, check the XForm with ODK Validate + :param pretty_print: If True, format the XForm with spaces, line breaks, etc. + :param enketo: If True, check the XForm with Enketo Validate. + :param form_name: Used for the main instance root node name. + :param default_language: The name of the default language for the form. + :param file_type: If provided, attempt parsing the data only as this type. Otherwise, + parsing of supported data types will be attempted until one of them succeeds. If the + xlsform is provided as a dict, then it is used directly and this argument is ignored. + """ + warnings = coalesce(warnings, []) + if isinstance(xlsform, dict): + workbook_dict = xlsform + fallback_form_name = None + else: + definition = get_definition_data(definition=xlsform) + if file_type is None: + file_type = definition.file_type + workbook_dict = definition_to_dict(definition=definition, file_type=file_type) + fallback_form_name = definition.file_path_stem + pyxform_data = xls2json.workbook_to_json( + workbook_dict=workbook_dict, + form_name=form_name, + fallback_form_name=fallback_form_name, + default_language=default_language, + warnings=warnings, + ) + survey = builder.create_survey_element_from_dict(pyxform_data) + xform = survey.to_xml( validate=validate, pretty_print=pretty_print, warnings=warnings, enketo=enketo, ) - output_dir = os.path.split(xform_path)[0] - if has_external_choices(json_survey): - itemsets_csv = os.path.join(output_dir, "itemsets.csv") - choices_exported = sheet_to_csv(xlsform_path, itemsets_csv, "external_choices") - if not choices_exported: - warnings.append( - "Could not export itemsets.csv, perhaps the " - "external choices sheet is missing." - ) - else: - logger.info("External choices csv is located at: %s", itemsets_csv) + itemsets = None + if has_external_choices(json_struct=pyxform_data): + itemsets = external_choices_to_csv(workbook_dict=workbook_dict) + return ConvertResult( + xform=xform, + warnings=warnings, + itemsets=itemsets, + _pyxform=pyxform_data, + _survey=survey, + ) + + +def xls2xform_convert( + xlsform_path: str | PathLike[str], + xform_path: str | PathLike[str], + validate: bool = True, + pretty_print: bool = True, + enketo: bool = False, +) -> list[str]: + warnings = [] + result = convert( + xlsform=xlsform_path, + validate=validate, + pretty_print=pretty_print, + enketo=enketo, + warnings=warnings, + ) + with open(xform_path, mode="w", encoding="utf-8") as f: + f.write(result.xform) + if result.itemsets is not None: + itemsets_path = Path(xform_path).parent / "itemsets.csv" + with open(itemsets_path, mode="w", encoding="utf-8", newline="") as f: + f.write(result.itemsets) + logger.info("External choices csv is located at: %s", itemsets_path) return warnings @@ -179,7 +272,7 @@ def main_cli(): logger.exception("EnvironmentError during conversion") except ODKValidateError: # Remove output file if there is an error - os.remove(args.output_path) + Path(args.output_path).unlink(missing_ok=True) logger.exception("ODKValidateError during conversion.") else: if len(warnings) > 0: diff --git a/tests/example_xls/calculate_without_calculation.xls b/tests/bug_example_xls/calculate_without_calculation.xls similarity index 100% rename from tests/example_xls/calculate_without_calculation.xls rename to tests/bug_example_xls/calculate_without_calculation.xls diff --git a/tests/example_xls/duplicate_columns.xlsx b/tests/bug_example_xls/duplicate_columns.xlsx similarity index 100% rename from tests/example_xls/duplicate_columns.xlsx rename to tests/bug_example_xls/duplicate_columns.xlsx diff --git a/tests/bug_example_xls/default_time_demo.xls b/tests/example_xls/default_time_demo.xls similarity index 100% rename from tests/bug_example_xls/default_time_demo.xls rename to tests/example_xls/default_time_demo.xls diff --git a/tests/example_xls/group.csv b/tests/example_xls/group.csv index 710d4d280..b8d4c4a5e 100644 --- a/tests/example_xls/group.csv +++ b/tests/example_xls/group.csv @@ -1,5 +1,5 @@ "survey",,, -,"type","name","label:English" +,"type","name","label:English (en)" ,"text","family_name","What's your family name?" ,"begin group","father","Father" ,"phone number","phone_number","What's your father's phone number?" diff --git a/tests/example_xls/group.md b/tests/example_xls/group.md new file mode 100644 index 000000000..d492fc928 --- /dev/null +++ b/tests/example_xls/group.md @@ -0,0 +1,7 @@ +| survey | +| | type | name | label:English (en) | +| | text | family_name | What's your family name? | +| | begin group | father | Father | +| | phone number | phone_number | What's your father's phone number? | +| | integer | age | How old is your father? | +| | end group | | | diff --git a/tests/example_xls/group.xls b/tests/example_xls/group.xls index b4958e24d..7b89251ba 100644 Binary files a/tests/example_xls/group.xls and b/tests/example_xls/group.xls differ diff --git a/tests/example_xls/group.xlsx b/tests/example_xls/group.xlsx index 064e6e94f..acfd2c170 100644 Binary files a/tests/example_xls/group.xlsx and b/tests/example_xls/group.xlsx differ diff --git a/tests/pyxform_test_case.py b/tests/pyxform_test_case.py index 1ca027f9c..df2fb5c75 100644 --- a/tests/pyxform_test_case.py +++ b/tests/pyxform_test_case.py @@ -3,11 +3,10 @@ """ import logging -import os -import re import tempfile from collections.abc import Iterable from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING, Optional from unittest import TestCase @@ -15,14 +14,11 @@ # noinspection PyProtectedMember from lxml.etree import _Element -from pyxform.builder import create_survey_element_from_dict from pyxform.constants import NSMAP from pyxform.errors import PyXFormError from pyxform.utils import coalesce from pyxform.validators.odk_validate import ODKValidateError, check_xform -from pyxform.xls2json import workbook_to_json - -from tests.test_utils.md_table import md_table_to_ss_structure +from pyxform.xls2xform import ConvertResult, convert logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler()) @@ -47,123 +43,25 @@ class MatcherContext: content_str: str -class PyxformMarkdown: - """Transform markdown formatted xlsform to a pyxform survey object""" - - def md_to_pyxform_survey( - self, - md_raw: str, - name: str | None = None, - title: str | None = None, - id_string: str | None = None, - debug: bool = False, - autoname: bool = True, - warnings: list[str] | None = None, - ): - if autoname: - kwargs = self._autoname_inputs(name=name, title=title, id_string=id_string) - name = kwargs["name"] - title = kwargs["title"] - id_string = kwargs["id_string"] - _md = [] - for line in md_raw.split("\n"): - if re.match(r"^\s+#", line): - # ignore lines which start with pound sign - continue - elif re.match(r"^(.*)(#[^|]+)$", line): - # keep everything before the # outside of the last occurrence - # of | - _md.append(re.match(r"^(.*)(#[^|]+)$", line).groups()[0].strip()) - else: - _md.append(line.strip()) - md = "\n".join(_md) - - if debug: - logger.debug(md) - - def list_to_dicts(arr): - headers = arr[0] - - def _row_to_dict(row): - out_dict = {} - for i in range(len(row)): - col = row[i] - if col not in [None, ""]: - out_dict[headers[i]] = col - return out_dict - - return [_row_to_dict(r) for r in arr[1:]] - - sheets = {} - for sheet, contents in md_table_to_ss_structure(md): - sheets[sheet] = list_to_dicts(contents) - - return self._ss_structure_to_pyxform_survey( - ss_structure=sheets, - name=name, - title=title, - id_string=id_string, - warnings=warnings, - ) - - @staticmethod - def _ss_structure_to_pyxform_survey( - ss_structure: dict, - name: str | None = None, - title: str | None = None, - id_string: str | None = None, - warnings: list[str] | None = None, - ): - # using existing methods from the builder - imported_survey_json = workbook_to_json( - workbook_dict=ss_structure, form_name=name, warnings=warnings - ) - # ideally, when all these tests are working, this would be refactored as well - survey = create_survey_element_from_dict(imported_survey_json) - # Due to str(name) in workbook_to_json - if survey.name is None or survey.name == "None": - survey.name = coalesce(name, "data") - if survey.title is None: - survey.title = title - if survey.id_string is None: - survey.id_string = id_string - - return survey - - @staticmethod - def _run_odk_validate(xml): - # On Windows, NamedTemporaryFile must be opened exclusively. - # So it must be explicitly created, opened, closed, and removed - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - try: - with open(tmp.name, mode="w", encoding="utf-8") as fp: - fp.write(xml) - fp.close() - check_xform(tmp.name) - finally: - # Clean up the temporary file - os.remove(tmp.name) - if os.path.isfile(tmp.name): - raise PyXFormError(f"Temporary file still exists: {tmp.name}") - - @staticmethod - def _autoname_inputs( - name: str | None = None, - title: str | None = None, - id_string: str | None = None, - ) -> dict[str, str]: - """ - Fill in any blank inputs with default values. - """ - return { - "name": coalesce(name, "test_name"), - "title": coalesce(title, "test_title"), - "id_string": coalesce(id_string, "test_id"), - } - - -class PyxformTestCase(PyxformMarkdown, TestCase): +def _run_odk_validate(xml): + # On Windows, NamedTemporaryFile must be opened exclusively. + # So it must be explicitly created, opened, closed, and removed + tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) + tmp.close() + try: + with open(tmp.name, mode="w", encoding="utf-8") as fp: + fp.write(xml) + fp.close() + check_xform(tmp.name) + finally: + # Clean up the temporary file + tmp_path = Path(tmp.name) + tmp_path.unlink(missing_ok=True) + if tmp_path.is_file(): + raise PyXFormError(f"Temporary file still exists: {tmp.name}") + + +class PyxformTestCase(TestCase): maxDiff = None def assertPyxformXform( @@ -194,12 +92,10 @@ def assertPyxformXform( errored: bool = False, # Optional extras name: str | None = None, - id_string: str | None = None, - title: str | None = None, warnings: list[str] | None = None, run_odk_validate: bool = False, debug: bool = False, - ): + ) -> ConvertResult | None: """ One survey input: :param md: a markdown formatted xlsform (easy to read in code). Escape a literal @@ -248,8 +144,6 @@ def assertPyxformXform( Optional extra parameters: :param name: a valid xml tag, for the root element in the XForm main instance. - :param id_string: an identifier, for the XForm main instance @id attribute. - :param title: a name, for the XForm header title. :param warnings: a list to use for storing warnings for inspection. :param run_odk_validate: If True, run ODK Validate on the XForm output. :param debug: If True, log details of the test to stdout. Details include the @@ -261,20 +155,16 @@ def assertPyxformXform( odk_validate_error__contains = coalesce(odk_validate_error__contains, []) survey_valid = True + result = None try: - if md is not None: - survey = self.md_to_pyxform_survey( - md_raw=md, + if survey is None: + result = convert( + xlsform=coalesce(md, ss_structure), + form_name=coalesce(name, "test_name"), warnings=warnings, - **self._autoname_inputs(name=name, title=title, id_string=id_string), - ) - elif ss_structure is not None: - survey = self._ss_structure_to_pyxform_survey( - ss_structure=ss_structure, - warnings=warnings, - **self._autoname_inputs(name=name, title=title, id_string=id_string), ) + survey = result._survey xml = survey._to_pretty_xml() root = etree.fromstring(xml.encode("utf-8")) @@ -310,7 +200,7 @@ def _pull_xml_node_from_root(element_selector): if debug: logger.debug(xml) if run_odk_validate: - self._run_odk_validate(xml=xml) + _run_odk_validate(xml=xml) if odk_validate_error__contains: raise PyxformTestError("ODKValidateError was not raised") @@ -423,6 +313,8 @@ def get_xpath_matcher_context(): raise PyxformTestError("warnings_count must be an integer.") self.assertEqual(warnings_count, len(warnings)) + return result + @staticmethod def _assert_contains(content, text, msg_prefix): if msg_prefix: diff --git a/tests/test_area.py b/tests/test_area.py index e09bb3e3e..58ebf58fb 100644 --- a/tests/test_area.py +++ b/tests/test_area.py @@ -17,7 +17,6 @@ def test_area(self): "38.25146813817506 21.758421137528785" ) self.assertPyxformXform( - name="area", md=f""" | survey | | | | | | | | type | name | label | calculation | default | @@ -25,8 +24,8 @@ def test_area(self): | | calculate | result | | enclosed-area(${{geoshape1}}) | | """, xml__xpath_match=[ - "/h:html/h:head/x:model/x:bind[@calculate='enclosed-area( /area/geoshape1 )' " - + " and @nodeset='/area/result' and @type='string']", - "/h:html/h:head/x:model/x:instance/x:area[x:geoshape1]", + "/h:html/h:head/x:model/x:bind[@calculate='enclosed-area( /test_name/geoshape1 )' " + + " and @nodeset='/test_name/result' and @type='string']", + "/h:html/h:head/x:model/x:instance/x:test_name[x:geoshape1]", ], ) diff --git a/tests/test_builder.py b/tests/test_builder.py index dc20f7c1c..83e10372a 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -4,6 +4,7 @@ import os import re +from pathlib import Path from unittest import TestCase import defusedxml.ElementTree as ETree @@ -59,8 +60,7 @@ def test_create_from_file_object(): def tearDown(self): fixture_path = utils.path_to_text_fixture("how_old_are_you.json") - if os.path.exists(fixture_path): - os.remove(fixture_path) + Path(fixture_path).unlink(missing_ok=True) def test_create_table_from_dict(self): d = { @@ -547,7 +547,7 @@ def test_style_column(self): def test_style_not_added_to_body_if_not_present(self): survey = utils.create_survey_from_fixture("widgets", filetype=FIXTURE_FILETYPE) - xml = survey.to_xml() + xml = survey.to_xml(pretty_print=False) # find the body tag root_elm = ETree.fromstring(xml.encode("utf-8")) body_elms = [ diff --git a/tests/test_dump_and_load.py b/tests/test_dump_and_load.py index f114d45d5..45702e513 100644 --- a/tests/test_dump_and_load.py +++ b/tests/test_dump_and_load.py @@ -3,6 +3,7 @@ """ import os +from pathlib import Path from unittest import TestCase from pyxform.builder import create_survey_from_path @@ -41,5 +42,5 @@ def test_load_from_dump(self): def tearDown(self): for survey in self.surveys.values(): - path = survey.name + ".json" - os.remove(path) + path = Path(survey.name + ".json") + path.unlink(missing_ok=True) diff --git a/tests/test_dynamic_default.py b/tests/test_dynamic_default.py index 3cf680579..595dd31e5 100644 --- a/tests/test_dynamic_default.py +++ b/tests/test_dynamic_default.py @@ -80,7 +80,7 @@ def model(q_num: int, case: Case): value_cmp = f"""and @value="{q_default_final}" """ return rf""" /h:html/h:head/x:model - /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ + /x:instance/x:test_name[@id="data"]/x:q{q_num}[ not(text()) and ancestor::x:model/x:bind[ @nodeset='/test_name/q{q_num}' @@ -102,7 +102,7 @@ def model(q_num: int, case: Case): q_default_cmp = f"""and text()='{q_default_final}' """ return rf""" /h:html/h:head/x:model - /x:instance/x:test_name[@id="test_id"]/x:q{q_num}[ + /x:instance/x:test_name[@id="data"]/x:q{q_num}[ ancestor::x:model/x:bind[ @nodeset='/test_name/q{q_num}' and @type='{q_bind}' @@ -169,7 +169,7 @@ def test_static_default_in_repeat(self): # Repeat template and first row. """ /h:html/h:head/x:model/x:instance/x:test_name[ - @id="test_id" + @id="data" and ./x:r1[@jr:template=''] and ./x:r1[not(@jr:template)] ] @@ -178,13 +178,13 @@ def test_static_default_in_repeat(self): """ /h:html/h:head/x:model[ ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] - ]/x:instance/x:test_name[@id="test_id"]/x:r1[@jr:template='']/x:q1[text()='12'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q1[text()='12'] """, # q1 static default value in repeat row. """ /h:html/h:head/x:model[ ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] - ]/x:instance/x:test_name[@id="test_id"]/x:r1[not(@jr:template)]/x:q1[text()='12'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q1[text()='12'] """, ], ) @@ -200,14 +200,12 @@ def test_dynamic_default_in_repeat(self): | | end repeat | r1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # Repeat template and first row. """ - /h:html/h:head/x:model/x:instance/x:test[ - @id="test" + /h:html/h:head/x:model/x:instance/x:test_name[ + @id="data" and ./x:r1[@jr:template=''] and ./x:r1[not(@jr:template)] ] @@ -215,39 +213,39 @@ def test_dynamic_default_in_repeat(self): # q0 dynamic default value not in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:q0[not(text())] + ./x:bind[@nodeset='/test_name/r1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q0[not(text())] """, # q0 dynamic default value not in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:q0[not(text())] + ./x:bind[@nodeset='/test_name/r1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q0[not(text())] """, # q0 dynamic default value not in model setvalue. """ - /h:html/h:head/x:model[not(./x:setvalue[@ref='test/r1/q0'])] + /h:html/h:head/x:model[not(./x:setvalue[@ref='data/r1/q0'])] """, # q0 dynamic default value in body group setvalue, with 2 events. """ - /h:html/h:body/x:group[@ref='/test/r1']/x:repeat[@nodeset='/test/r1'] + /h:html/h:body/x:group[@ref='/test_name/r1']/x:repeat[@nodeset='/test_name/r1'] /x:setvalue[ @event='odk-instance-first-load odk-new-repeat' - and @ref='/test/r1/q0' + and @ref='/test_name/r1/q0' and @value='random()' ] """, # q1 static default value in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='string'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:q1[text()='not_func$'] + ./x:bind[@nodeset='/test_name/r1/q1' and @type='string'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q1[text()='not_func$'] """, # q1 static default value in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='string'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:q1[text()='not_func$'] + ./x:bind[@nodeset='/test_name/r1/q1' and @type='string'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q1[text()='not_func$'] """, ], ) @@ -263,29 +261,27 @@ def test_dynamic_default_in_group(self): | | end group | g1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # q0 element in instance. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:q0""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:q0""", # Group element in instance. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:g1""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:g1""", # q1 dynamic default not in instance. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:g1/x:q1[not(text())]""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:g1/x:q1[not(text())]""", # q1 dynamic default value in model setvalue, with 1 event. """ /h:html/h:head/x:model/x:setvalue[ @event="odk-instance-first-load" - and @ref='/test/g1/q1' - and @value=' /test/q0 ' + and @ref='/test_name/g1/q1' + and @value=' /test_name/q0 ' ] """, # q1 dynamic default value not in body group setvalue. """ /h:html/h:body/x:group[ - @ref='/test/g1' - and not(child::setvalue[@ref='/test/g1/q1']) + @ref='/test_name/g1' + and not(child::setvalue[@ref='/test_name/g1/q1']) ] """, ], @@ -302,29 +298,27 @@ def test_sibling_dynamic_default_in_group(self): | | end group | g1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # Group element in instance. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:g1""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:g1""", # q0 element in group. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:g1/x:q0""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:g1/x:q0""", # q1 dynamic default not in instance. - """/h:html/h:head/x:model/x:instance/x:test[@id="test"]/x:g1/x:q1[not(text())]""", + """/h:html/h:head/x:model/x:instance/x:test_name[@id="data"]/x:g1/x:q1[not(text())]""", # q1 dynamic default value in model setvalue, with 1 event. """ /h:html/h:head/x:model/x:setvalue[ @event="odk-instance-first-load" - and @ref='/test/g1/q1' - and @value=' /test/g1/q0 ' + and @ref='/test_name/g1/q1' + and @value=' /test_name/g1/q0 ' ] """, # q1 dynamic default value not in body group setvalue. """ /h:html/h:body/x:group[ - @ref='/test/g1' - and not(child::setvalue[@ref='/test/g1/q1']) + @ref='/test_name/g1' + and not(child::setvalue[@ref='/test_name/g1/q1']) ] """, ], @@ -341,14 +335,12 @@ def test_sibling_dynamic_default_in_repeat(self): | | end repeat | r1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # Repeat template and first row. """ - /h:html/h:head/x:model/x:instance/x:test[ - @id="test" + /h:html/h:head/x:model/x:instance/x:test_name[ + @id="data" and ./x:r1[@jr:template=''] and ./x:r1[not(@jr:template)] ] @@ -356,25 +348,25 @@ def test_sibling_dynamic_default_in_repeat(self): # q0 element in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:q0 + ./x:bind[@nodeset='/test_name/r1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q0 """, # q0 element in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:q0 + ./x:bind[@nodeset='/test_name/r1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q0 """, # q1 dynamic default value not in model setvalue. """ - /h:html/h:head/x:model[not(./x:setvalue[@ref='test/r1/q1'])] + /h:html/h:head/x:model[not(./x:setvalue[@ref='data/r1/q1'])] """, # q1 dynamic default value in body group setvalue, with 2 events. """ - /h:html/h:body/x:group[@ref='/test/r1']/x:repeat[@nodeset='/test/r1'] + /h:html/h:body/x:group[@ref='/test_name/r1']/x:repeat[@nodeset='/test_name/r1'] /x:setvalue[ @event='odk-instance-first-load odk-new-repeat' - and @ref='/test/r1/q1' + and @ref='/test_name/r1/q1' and @value=' ../q0 ' ] """, @@ -394,14 +386,12 @@ def test_dynamic_default_in_group_nested_in_repeat(self): | | end repeat | r1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # Repeat template and first row contains the group. """ - /h:html/h:head/x:model/x:instance/x:test[ - @id="test" + /h:html/h:head/x:model/x:instance/x:test_name[ + @id="data" and ./x:r1[@jr:template='']/x:g1 and ./x:r1[not(@jr:template)]/x:g1 ] @@ -409,25 +399,25 @@ def test_dynamic_default_in_group_nested_in_repeat(self): # q0 element in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/g1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:g1/x:q0 + ./x:bind[@nodeset='/test_name/r1/g1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:g1/x:q0 """, # q0 element in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/g1/q0' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:g1/x:q0 + ./x:bind[@nodeset='/test_name/r1/g1/q0' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:g1/x:q0 """, # q1 dynamic default value not in model setvalue. """ - /h:html/h:head/x:model[not(./x:setvalue[@ref='test/r1/g1/q1'])] + /h:html/h:head/x:model[not(./x:setvalue[@ref='data/r1/g1/q1'])] """, # q1 dynamic default value in body group setvalue, with 2 events. """ - /h:html/h:body/x:group[@ref='/test/r1']/x:repeat[@nodeset='/test/r1'] + /h:html/h:body/x:group[@ref='/test_name/r1']/x:repeat[@nodeset='/test_name/r1'] /x:setvalue[ @event='odk-instance-first-load odk-new-repeat' - and @ref='/test/r1/g1/q1' + and @ref='/test_name/r1/g1/q1' and @value=' ../q0 ' ] """, @@ -448,14 +438,12 @@ def test_dynamic_default_in_repeat_nested_in_repeat(self): | | end repeat | r1 | | | """ self.assertPyxformXform( - name="test", - id_string="test", md=md, xml__xpath_match=[ # Repeat templates and first rows. """ - /h:html/h:head/x:model/x:instance/x:test[ - @id="test" + /h:html/h:head/x:model/x:instance/x:test_name[ + @id="data" and ./x:r1[@jr:template='']/x:r2[@jr:template=''] and ./x:r1[not(@jr:template)]/x:r2[not(@jr:template)] ] @@ -463,63 +451,63 @@ def test_dynamic_default_in_repeat_nested_in_repeat(self): # q0 element in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='date'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:q0 + ./x:bind[@nodeset='/test_name/r1/q0' and @type='date'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q0 """, # q0 element in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q0' and @type='date'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:q0 + ./x:bind[@nodeset='/test_name/r1/q0' and @type='date'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q0 """, # q0 dynamic default value not in model setvalue. """ - /h:html/h:head/x:model[not(./x:setvalue[@ref='test/r1/q0'])] + /h:html/h:head/x:model[not(./x:setvalue[@ref='data/r1/q0'])] """, # q0 dynamic default value in body group setvalue, with 2 events. """ - /h:html/h:body/x:group[@ref='/test/r1']/x:repeat[@nodeset='/test/r1'] + /h:html/h:body/x:group[@ref='/test_name/r1']/x:repeat[@nodeset='/test_name/r1'] /x:setvalue[ @event='odk-instance-first-load odk-new-repeat' - and @ref='/test/r1/q0' + and @ref='/test_name/r1/q0' and @value='now()' ] """, # q1 element in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:q1 + ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:q1 """, # q1 element in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:q1 + ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:q1 """, # q2 element in repeat template. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[@jr:template='']/x:r2[@jr:template='']/x:q2 + ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[@jr:template='']/x:r2[@jr:template='']/x:q2 """, # q2 element in repeat row. """ /h:html/h:head/x:model[ - ./x:bind[@nodeset='/test/r1/q1' and @type='int'] - ]/x:instance/x:test[@id="test"]/x:r1[not(@jr:template)]/x:r2[not(@jr:template)]/x:q2 + ./x:bind[@nodeset='/test_name/r1/q1' and @type='int'] + ]/x:instance/x:test_name[@id="data"]/x:r1[not(@jr:template)]/x:r2[not(@jr:template)]/x:q2 """, # q2 dynamic default value not in model setvalue. """ - /h:html/h:head/x:model[not(./x:setvalue[@ref='test/r1/r2/q2'])] + /h:html/h:head/x:model[not(./x:setvalue[@ref='data/r1/r2/q2'])] """, # q2 dynamic default value in body group setvalue, with 2 events. """ - /h:html/h:body/x:group[@ref='/test/r1']/x:repeat[@nodeset='/test/r1'] - /x:group[@ref='/test/r1/r2']/x:repeat[@nodeset='/test/r1/r2'] + /h:html/h:body/x:group[@ref='/test_name/r1']/x:repeat[@nodeset='/test_name/r1'] + /x:group[@ref='/test_name/r1/r2']/x:repeat[@nodeset='/test_name/r1/r2'] /x:setvalue[ @event='odk-instance-first-load odk-new-repeat' - and @ref='/test/r1/r2/q2' + and @ref='/test_name/r1/r2/q2' and @value=' ../../q1 ' ] """, @@ -548,7 +536,7 @@ def test_dynamic_default_on_calculate(self): """, xml__xpath_match=[ xp.model(1, Case(True, "calculate", "random() + 0.5")), - xp.model(2, Case(True, "calculate", "if( /test/q1 < 1,'A','B')")), + xp.model(2, Case(True, "calculate", "if( /test_name/q1 < 1,'A','B')")), # Nothing in body since both questions are calculations. "/h:html/h:body[not(text) and count(./*) = 0]", ], diff --git a/tests/test_external_instances.py b/tests/test_external_instances.py index e0d33ab92..a206aaa0e 100644 --- a/tests/test_external_instances.py +++ b/tests/test_external_instances.py @@ -6,8 +6,6 @@ from textwrap import dedent -from pyxform.errors import PyXFormError - from tests.pyxform_test_case import PyxformTestCase, PyxformTestError from tests.xpath_helpers.choices import xpc @@ -249,15 +247,17 @@ def test_cannot__use_different_src_same_id__select_then_internal(self): | | states | 1 | Pass | | | | states | 2 | Fail | | """ - with self.assertRaises(PyXFormError) as ctx: - survey = self.md_to_pyxform_survey(md_raw=md) - survey._to_pretty_xml() - self.assertIn( - "Instance name: 'states', " - "Existing type: 'file', Existing URI: 'jr://file-csv/states.csv', " - "Duplicate type: 'choice', Duplicate URI: 'None', " - "Duplicate context: 'survey'.", - repr(ctx.exception), + self.assertPyxformXform( + md=md, + errored=True, + error__contains=[ + ( + "Instance name: 'states', " + "Existing type: 'file', Existing URI: 'jr://file-csv/states.csv', " + "Duplicate type: 'choice', Duplicate URI: 'None', " + "Duplicate context: 'survey'." + ) + ], ) def test_cannot__use_different_src_same_id__external_then_pulldata(self): @@ -273,15 +273,17 @@ def test_cannot__use_different_src_same_id__external_then_pulldata(self): | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | """ - with self.assertRaises(PyXFormError) as ctx: - survey = self.md_to_pyxform_survey(md_raw=md) - survey._to_pretty_xml() - self.assertIn( - "Instance name: 'fruits', " - "Existing type: 'external', Existing URI: 'jr://file/fruits.xml', " - "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " - "Duplicate context: '[type: group, name: g1]'.", - repr(ctx.exception), + self.assertPyxformXform( + md=md, + errored=True, + error__contains=[ + ( + "Instance name: 'fruits', " + "Existing type: 'external', Existing URI: 'jr://file/fruits.xml', " + "Duplicate type: 'pulldata', Duplicate URI: 'jr://file-csv/fruits.csv', " + "Duplicate context: '[type: group, name: g1]'." + ) + ], ) def test_cannot__use_different_src_same_id__pulldata_then_external(self): @@ -297,15 +299,17 @@ def test_cannot__use_different_src_same_id__pulldata_then_external(self): | | note | note | Fruity! ${f_csv} | | | | end group | g1 | | | """ - with self.assertRaises(PyXFormError) as ctx: - survey = self.md_to_pyxform_survey(md_raw=md) - survey._to_pretty_xml() - self.assertIn( - "Instance name: 'fruits', " - "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " - "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " - "Duplicate context: '[type: group, name: g1]'.", - repr(ctx.exception), + self.assertPyxformXform( + md=md, + errored=True, + error__contains=[ + ( + "Instance name: 'fruits', " + "Existing type: 'pulldata', Existing URI: 'jr://file-csv/fruits.csv', " + "Duplicate type: 'external', Duplicate URI: 'jr://file/fruits.xml', " + "Duplicate context: '[type: group, name: g1]'." + ) + ], ) def test_can__reuse_csv__selects_then_pulldata(self): @@ -320,13 +324,17 @@ def test_can__reuse_csv__selects_then_pulldata(self): | | calculate | f_csv | pd | pulldata('pain_locations', 'name', 'name', 'arm') | | | note | note | Arm ${f_csv} | | """ - expected = """ - -""" - self.assertPyxformXform(md=md, model__contains=[expected]) - survey = self.md_to_pyxform_survey(md_raw=md) - xml = survey._to_pretty_xml() - self.assertEqual(1, xml.count(expected)) + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='pain_locations' + and @src='jr://file-csv/pain_locations.csv' + ] + """ + ], + ) def test_can__reuse_csv__pulldata_then_selects(self): """Re-using the same csv external data source id and URI is OK.""" @@ -340,10 +348,17 @@ def test_can__reuse_csv__pulldata_then_selects(self): | | select_one_from_file pain_locations.csv | pmonth | Location of worst pain this month. | | | | select_one_from_file pain_locations.csv | pyear | Location of worst pain this year. | | """ - expected = ( - """""" + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='pain_locations' + and @src='jr://file-csv/pain_locations.csv' + ] + """ + ], ) - self.assertPyxformXform(md=md, model__contains=[expected]) def test_can__reuse_xml__selects_then_external(self): """Re-using the same xml external data source id and URI is OK.""" @@ -356,12 +371,17 @@ def test_can__reuse_xml__selects_then_external(self): | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | | | xml-external | pain_locations | | """ - expected = """ - -""" - survey = self.md_to_pyxform_survey(md_raw=md) - xml = survey._to_pretty_xml() - self.assertEqual(1, xml.count(expected)) + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='pain_locations' + and @src='jr://file/pain_locations.xml' + ] + """ + ], + ) def test_can__reuse_xml__external_then_selects(self): """Re-using the same xml external data source id and URI is OK.""" @@ -374,13 +394,17 @@ def test_can__reuse_xml__external_then_selects(self): | | select_one_from_file pain_locations.xml | pmonth | Location of worst pain this month. | | | select_one_from_file pain_locations.xml | pyear | Location of worst pain this year. | """ - expected = ( - """""" + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='pain_locations' + and @src='jr://file/pain_locations.xml' + ] + """ + ], ) - self.assertPyxformXform(md=md, model__contains=[expected]) - survey = self.md_to_pyxform_survey(md_raw=md) - xml = survey._to_pretty_xml() - self.assertEqual(1, xml.count(expected)) def test_external_instance_pulldata_constraint(self): """ @@ -570,10 +594,17 @@ def test_external_instance_pulldata(self): | | type | name | label | relevant | required | constraint | | | text | Part_ID | Participant ID | pulldata('ID', 'ParticipantID', 'ParticipantIDValue',.) | pulldata('ID', 'ParticipantID', 'ParticipantIDValue',.) | pulldata('ID', 'ParticipantID', 'ParticipantIDValue',.) | """ - node = """""" - survey = self.md_to_pyxform_survey(md_raw=md) - xml = survey._to_pretty_xml() - self.assertEqual(1, xml.count(node)) + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='ID' + and @src='jr://file-csv/ID.csv' + ] + """ + ], + ) def test_external_instances_multiple_diff_pulldatas(self): """ @@ -583,17 +614,27 @@ def test_external_instances_multiple_diff_pulldatas(self): columns but pulling data from different csv files """ md = """ - | survey | | | | | | - | | type | name | label | relevant | required | - | | text | Part_ID | Participant ID | pulldata('fruits', 'name', 'name_key', 'mango') | pulldata('OtherID', 'ParticipantID', ParticipantIDValue',.) | + | survey | | | | | | + | | type | name | label | relevant | required | + | | text | Part_ID | Participant ID | pulldata('fruits', 'name', 'name_key', 'mango') | pulldata('OtherID', 'ParticipantID', ParticipantIDValue',.) | """ - node1 = '' - node2 = '' - - survey = self.md_to_pyxform_survey(md_raw=md) - xml = survey._to_pretty_xml() - self.assertEqual(1, xml.count(node1)) - self.assertEqual(1, xml.count(node2)) + self.assertPyxformXform( + md=md, + xml__xpath_match=[ + """ + /h:html/h:head/x:model/x:instance[ + @id='fruits' + and @src='jr://file-csv/fruits.csv' + ] + """, + """ + /h:html/h:head/x:model/x:instance[ + @id='OtherID' + and @src='jr://file-csv/OtherID.csv' + ] + """, + ], + ) def test_mixed_quotes_and_functions_in_pulldata(self): # re: https://github.com/XLSForm/pyxform/issues/398 diff --git a/tests/test_external_instances_for_selects.py b/tests/test_external_instances_for_selects.py index b49aa9eb4..75ee04a90 100644 --- a/tests/test_external_instances_for_selects.py +++ b/tests/test_external_instances_for_selects.py @@ -10,10 +10,10 @@ from pyxform import aliases from pyxform.constants import EXTERNAL_INSTANCE_EXTENSIONS from pyxform.errors import PyXFormError +from pyxform.xls2json_backends import md_table_to_workbook from pyxform.xls2xform import get_xml_path, xls2xform_convert from tests.pyxform_test_case import PyxformTestCase -from tests.test_utils.md_table import md_table_to_workbook from tests.utils import get_temp_dir from tests.xpath_helpers.choices import xpc from tests.xpath_helpers.questions import xpq diff --git a/tests/test_fieldlist_labels.py b/tests/test_fieldlist_labels.py index bd7d5be78..469fd3a45 100644 --- a/tests/test_fieldlist_labels.py +++ b/tests/test_fieldlist_labels.py @@ -9,101 +9,77 @@ class FieldListLabels(PyxformTestCase): """Test unlabeled group""" def test_unlabeled_group(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | begin_group | my-group | | | | text | my-text | my-text | | | end_group | | | """, - warnings=warnings, + warnings_count=1, + warnings__contains=["[row : 2] Group has no label"], ) - self.assertTrue(len(warnings) == 1) - self.assertTrue("[row : 2] Group has no label" in warnings[0]) - def test_unlabeled_group_alternate_syntax(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label::English (en) | | | begin group | my-group | | | | text | my-text | my-text | | | end group | | | """, - warnings=warnings, + warnings_count=1, + warnings__contains=["[row : 2] Group has no label"], ) - self.assertTrue(len(warnings) == 1) - self.assertTrue("[row : 2] Group has no label" in warnings[0]) - def test_unlabeled_group_fieldlist(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | | type | name | label | appearance | | | begin_group | my-group | | field-list | | | text | my-text | my-text | | | | end_group | | | | """, - warnings=warnings, + warnings_count=0, ) - self.assertTrue(len(warnings) == 0) - def test_unlabeled_group_fieldlist_alternate_syntax(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | | type | name | label | appearance | | | begin group | my-group | | field-list | | | text | my-text | my-text | | | | end group | | | | """, - warnings=warnings, + warnings_count=0, ) - self.assertTrue(len(warnings) == 0) - def test_unlabeled_repeat(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | begin_repeat | my-repeat | | | | text | my-text | my-text | | | end_repeat | | | """, - warnings=warnings, + warnings_count=1, + warnings__contains=["[row : 2] Repeat has no label"], ) - self.assertTrue(len(warnings) == 1) - self.assertTrue("[row : 2] Repeat has no label" in warnings[0]) - def test_unlabeled_repeat_fieldlist(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | | type | name | label | appearance | | | begin_repeat | my-repeat | | field-list | | | text | my-text | my-text | | | | end_repeat | | | | """, - warnings=warnings, + warnings_count=1, + warnings__contains=["[row : 2] Repeat has no label"], ) - - self.assertTrue(len(warnings) == 1) - self.assertTrue("[row : 2] Repeat has no label" in warnings[0]) diff --git a/tests/test_form_name.py b/tests/test_form_name.py index 866fec600..e3f395307 100644 --- a/tests/test_form_name.py +++ b/tests/test_form_name.py @@ -7,33 +7,17 @@ class TestFormName(PyxformTestCase): def test_default_to_data_when_no_name(self): - """ - Test no form_name will default to survey name to 'data'. - """ - survey = self.md_to_pyxform_survey( - """ + """Should default to form_name of 'test_name', and form id of 'data'.""" + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | text | city | City Name | """, - autoname=False, - ) - - # We're passing autoname false when creating the survey object. - self.assertEqual(survey.id_string, None) - self.assertEqual(survey.name, "data") - self.assertEqual(survey.title, None) - - # Set required fields because we need them if we want to do xml comparison. - survey.id_string = "some-id" - survey.title = "data" - - self.assertPyxformXform( - survey=survey, - instance__contains=[''], - model__contains=[''], + instance__contains=[''], + model__contains=[''], xml__contains=[ - '', + '', "", "", ], @@ -50,8 +34,7 @@ def test_default_to_data(self): | | text | city | City Name | """, name="data", - id_string="some-id", - instance__contains=[''], + instance__contains=[''], model__contains=[''], xml__contains=[ '', @@ -72,8 +55,7 @@ def test_default_form_name_to_superclass_definition(self): | | text | city | City Name | """, name="some-name", - id_string="some-id", - instance__contains=[''], + instance__contains=[''], model__contains=[''], xml__contains=[ '', diff --git a/tests/test_group.py b/tests/test_group.py index 9d99fbe50..7daa235f0 100644 --- a/tests/test_group.py +++ b/tests/test_group.py @@ -25,22 +25,24 @@ def test_json(self): { "name": "family_name", "type": "text", - "label": {"English": "What's your family name?"}, + "label": {"English (en)": "What's your family name?"}, }, { "name": "father", "type": "group", - "label": {"English": "Father"}, + "label": {"English (en)": "Father"}, "children": [ { "name": "phone_number", "type": "phone number", - "label": {"English": "What's your father's phone number?"}, + "label": { + "English (en)": "What's your father's phone number?" + }, }, { "name": "age", "type": "integer", - "label": {"English": "How old is your father?"}, + "label": {"English (en)": "How old is your father?"}, }, ], }, diff --git a/tests/test_image_app_parameter.py b/tests/test_image_app_parameter.py index eab62071d..8d65db32f 100644 --- a/tests/test_image_app_parameter.py +++ b/tests/test_image_app_parameter.py @@ -150,21 +150,20 @@ def test_string_extra_params(self): ) def test_image_with_no_max_pixels_should_warn(self): - warnings = [] - - self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | image | my_image | Image | | | image | my_image_1 | Image 1 | """, - warnings=warnings, + warnings_count=2, + warnings__contains=[ + "[row : 2] Use the max-pixels parameter to speed up submission sending and save storage space. Learn more: https://xlsform.org/#image", + "[row : 3] Use the max-pixels parameter to speed up submission sending and save storage space. Learn more: https://xlsform.org/#image", + ], ) - self.assertTrue(len(warnings) == 2) - self.assertTrue("max-pixels" in warnings[0] and "max-pixels" in warnings[1]) - def test_max_pixels_and_app(self): self.assertPyxformXform( name="data", diff --git a/tests/test_language_warnings.py b/tests/test_language_warnings.py index 9892d0ca3..ee255bf83 100644 --- a/tests/test_language_warnings.py +++ b/tests/test_language_warnings.py @@ -2,9 +2,6 @@ Test language warnings. """ -import os -import tempfile - from tests.pyxform_test_case import PyxformTestCase @@ -14,67 +11,46 @@ class LanguageWarningTest(PyxformTestCase): """ def test_label_with_valid_subtag_should_not_warn(self): - survey = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label::English (en) | | | note | my_note | My note | - """ + """, + warnings_count=0, ) - warnings = [] - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - - self.assertTrue(len(warnings) == 0) - os.unlink(tmp.name) - def test_label_with_no_subtag_should_warn(self): - survey = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label::English | | | note | my_note | My note | - """ - ) - - warnings = [] - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - - self.assertTrue(len(warnings) == 1) - self.assertTrue( - "do not contain valid machine-readable codes: English. Learn more" - in warnings[0] + """, + warnings_count=1, + warnings__contains=[ + "The following language declarations do not contain valid machine-readable " + "codes: English. Learn more: http://xlsform.org#multiple-language-support" + ], ) - os.unlink(tmp.name) def test_label_with_unknown_subtag_should_warn(self): - survey = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label::English (schm) | | | note | my_note | My note | - """ + """, + warnings_count=1, + warnings__contains=[ + "The following language declarations do not contain valid machine-readable " + "codes: English (schm). Learn more: http://xlsform.org#multiple-language-support" + ], ) - warnings = [] - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - - self.assertTrue(len(warnings) == 1) - self.assertTrue( - "do not contain valid machine-readable codes: English (schm). Learn more" - in warnings[0] - ) - os.unlink(tmp.name) - def test_default_language_only_should_not_warn(self): - survey = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | | type | name | label | choice_filter | | | select_one opts | opt | My opt | fake = 1 | @@ -82,13 +58,6 @@ def test_default_language_only_should_not_warn(self): | | list_name | name | label | fake | | | opts | opt1 | Opt1 | 1 | | | opts | opt2 | Opt2 | 1 | - """ + """, + warnings_count=0, ) - - warnings = [] - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - - self.assertTrue(len(warnings) == 0) - os.unlink(tmp.name) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 2861d06c3..e008dbe77 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -2,9 +2,6 @@ Test language warnings. """ -import os -import tempfile - from tests.pyxform_test_case import PyxformTestCase @@ -39,45 +36,31 @@ def test_metadata_bindings(self): ) def test_simserial_deprecation_warning(self): - warnings = [] - survey = self.md_to_pyxform_survey( - """ - | survey | | | | - | | type | name | label | - | | simserial | simserial | | - | | note | simserial_test_output | simserial_test_output: ${simserial} | + self.assertPyxformXform( + md=""" + | survey | | | | + | | type | name | label | + | | simserial | simserial | | + | | note | simserial_test_output | simserial_test_output: ${simserial} | """, - warnings=warnings, - ) - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - self.assertTrue(len(warnings) == 1) - warning_expected = ( - "[row : 2] simserial is no longer supported on most devices. " - "Only old versions of Collect on Android versions older than 11 still support it." + warnings_count=1, + warnings__contains=[ + "[row : 2] simserial is no longer supported on most devices. " + "Only old versions of Collect on Android versions older than 11 still support it." + ], ) - self.assertEqual(warning_expected, warnings[0]) - os.unlink(tmp.name) def test_subscriber_id_deprecation_warning(self): - warnings = [] - survey = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | subscriberid | subscriberid | sub id - extra warning generated w/o this | | | note | subscriberid_test_output | subscriberid_test_output: ${subscriberid} | """, - warnings=warnings, - ) - tmp = tempfile.NamedTemporaryFile(suffix=".xml", delete=False) - tmp.close() - survey.print_xform_to_file(tmp.name, warnings=warnings) - self.assertTrue(len(warnings) == 1) - warning_expected = ( - "[row : 2] subscriberid is no longer supported on most devices. " - "Only old versions of Collect on Android versions older than 11 still support it." + warnings_count=1, + warnings__contains=[ + "[row : 2] subscriberid is no longer supported on most devices. " + "Only old versions of Collect on Android versions older than 11 still support it." + ], ) - self.assertEqual(warning_expected, warnings[0]) - os.unlink(tmp.name) diff --git a/tests/test_pyxform_test_case.py b/tests/test_pyxform_test_case.py index ceea0c4bd..87fa566a1 100644 --- a/tests/test_pyxform_test_case.py +++ b/tests/test_pyxform_test_case.py @@ -104,7 +104,7 @@ class TestPyxformTestCaseXmlXpath(PyxformTestCase): exact={ ( """\n""" - """ \n""" + """ \n""" """ \n""" """ \n""" """ \n""" diff --git a/tests/test_pyxformtestcase.py b/tests/test_pyxformtestcase.py index f6e2599aa..3fb2ec3ab 100644 --- a/tests/test_pyxformtestcase.py +++ b/tests/test_pyxformtestcase.py @@ -56,14 +56,13 @@ def test_formid_is_not_none(self): When the form id is not set, it should never use python's None. Fixing because this messes up other tests. """ - s1 = self.md_to_pyxform_survey( - """ + self.assertPyxformXform( + md=""" | survey | | | | | | type | name | label | | | note | q | Q | """, - autoname=True, + xml__xpath_match=[ + "/h:html/h:head/x:model/x:instance/x:test_name[@id='data']" + ], ) - - if s1.id_string in ["None", None]: - self.assertRaises(Exception, lambda: s1.validate()) diff --git a/tests/test_repeat.py b/tests/test_repeat.py index aa06a28c6..93cd52fa0 100644 --- a/tests/test_repeat.py +++ b/tests/test_repeat.py @@ -15,8 +15,6 @@ def test_repeat_relative_reference(self): Test relative reference in repeats. """ self.assertPyxformXform( - name="test_repeat", - title="Relative Paths in repeats", md=""" | survey | | | | | | | type | name | relevant | label | @@ -72,27 +70,27 @@ def test_repeat_relative_reference(self): "", ], model__contains=[ - """""", - """""", + """""", - """""", - """""", - """""", - """""", ], xml__contains=[ - '', + '', "", "", """""", - """""", + """""", """""", ], @@ -100,9 +98,8 @@ def test_repeat_relative_reference(self): def test_calculate_relative_path(self): """Test relative paths in calculate column.""" + # Paths in a calculate within a repeat are relative. self.assertPyxformXform( - name="data", - title="Paths in a calculate within a repeat are relative.", md=""" | survey | | | | | | | type | name | label | calculation | @@ -122,17 +119,16 @@ def test_calculate_relative_path(self): """, # pylint: disable=line-too-long model__contains=[ """""", + """nodeset="/test_name/rep/a" type="string"/>""", """""", + """nodeset="/test_name/rep/group/b" type="string"/>""", ], ) - def test_choice_filter_relative_path(self): # pylint: disable=invalid-name + def test_choice_filter_relative_path(self): """Test relative paths in choice_filter column.""" + # Choice filter uses relative path self.assertPyxformXform( - name="data", - title="Choice filter uses relative path", md=""" | survey | | | | | | | type | name | label | choice_filter | @@ -158,9 +154,8 @@ def test_choice_filter_relative_path(self): # pylint: disable=invalid-name def test_indexed_repeat_relative_path(self): """Test relative path not used with indexed-repeat().""" + # Paths in a calculate within a repeat are relative. self.assertPyxformXform( - name="data", - title="Paths in a calculate within a repeat are relative.", md=""" | survey | | | | | | | type | name | label | calculation | @@ -183,7 +178,7 @@ def test_indexed_repeat_relative_path(self): | | crop_list | kale | Kale | | """, # pylint: disable=line-too-long model__contains=[ - """""" # pylint: disable=line-too-long + """""" # pylint: disable=line-too-long ], ) @@ -200,7 +195,6 @@ def test_output_with_translation_relative_path(self): self.assertPyxformXform( md=md, - name="inside-repeat-relative-path", xml__contains=[ '', ' Name of ', @@ -223,7 +217,6 @@ def test_output_with_guidance_hint_translation_relative_path(self): self.assertPyxformXform( md=md, - name="inside-repeat-relative-path", xml__contains=[ '', ' Name of ', @@ -244,7 +237,6 @@ def test_output_with_multiple_translations_relative_path(self): self.assertPyxformXform( md=md, - name="inside-repeat-relative-path", xml__contains=[ '', ' Name of ', @@ -320,10 +312,9 @@ def test_choice_from_previous_repeat_answers(self): | | select one ${name} | choice | Choose name | """ self.assertPyxformXform( - name="data", md=xlsform_md, xml__contains=[ - "", + "", '', '