From 60ab0a604661168d685f23a5d0548f844a5cb72f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Mar 2024 12:11:08 +0300 Subject: [PATCH 01/37] Django 4: initial update requirement packages --- requirements/azure.in | 4 +- requirements/azure.pip | 3 +- requirements/base.pip | 82 +++++++++++------------------ requirements/dev.in | 15 +++--- requirements/dev.pip | 117 +++++++++++++++++++---------------------- requirements/mysql.pip | 1 - requirements/s3.in | 4 +- requirements/s3.pip | 18 +++---- requirements/ses.in | 2 +- requirements/ses.pip | 20 +++---- setup.cfg | 4 +- 11 files changed, 115 insertions(+), 155 deletions(-) delete mode 100644 requirements/mysql.pip diff --git a/requirements/azure.in b/requirements/azure.in index b8799ecf47..fbae9b96a5 100644 --- a/requirements/azure.in +++ b/requirements/azure.in @@ -1,3 +1,3 @@ -django-storages[azure] cryptography>=39.0.1 -django >=3.2.25,<4 +django ==4.0,<5 +django-storages[azure] diff --git a/requirements/azure.pip b/requirements/azure.pip index 169326e3e8..8c8828a584 100644 --- a/requirements/azure.pip +++ b/requirements/azure.pip @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=requirements/azure.pip requirements/azure.in @@ -46,7 +46,6 @@ sqlparse==0.4.4 # via django typing-extensions==4.10.0 # via - # asgiref # azure-core # azure-storage-blob urllib3==2.2.1 diff --git a/requirements/base.pip b/requirements/base.pip index a75fd021b8..53b832d678 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/base.pip requirements/base.in +# pip-compile --output-file=requirements/base.pip --strip-extras requirements/base.in # -e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest # via -r requirements/base.in @@ -30,8 +30,6 @@ asgiref==3.7.2 # via # django # django-cors-headers -async-timeout==4.0.3 - # via redis attrs==23.2.0 # via # jsonlines @@ -43,11 +41,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.61 - # via - # dataflows-tabulator - # tabulator -botocore==1.34.61 +boto3==1.34.66 + # via dataflows-tabulator +botocore==1.34.66 # via # boto3 # s3transfer @@ -67,7 +63,6 @@ chardet==5.2.0 # via # dataflows-tabulator # datapackage - # tabulator charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -79,7 +74,6 @@ click==8.1.7 # dataflows-tabulator # datapackage # tableschema - # tabulator click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 @@ -91,8 +85,10 @@ cryptography==42.0.5 # jwcrypto # onadata # pyjwt -dataflows-tabulator==1.54.0 - # via datapackage +dataflows-tabulator==1.54.1 + # via + # datapackage + # tableschema datapackage==1.15.4 # via pyfloip defusedxml==0.7.1 @@ -104,7 +100,7 @@ deprecated==1.2.14 # via onadata dict2xml==1.7.5 # via onadata -django==3.2.25 +django==4.0 # via # django-activity-stream # django-cors-headers @@ -139,7 +135,7 @@ django-guardian==2.4.0 # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==2.3.0 +django-oauth-toolkit==2.1.0 # via onadata django-ordered-model==3.7.4 # via onadata @@ -157,7 +153,7 @@ django-taggit==4.0.0 # via onadata django-templated-email==3.0.1 # via onadata -djangorestframework==3.14.0 +djangorestframework==3.15.0 # via # djangorestframework-csv # djangorestframework-gis @@ -179,7 +175,7 @@ djangorestframework-xml==2.0.0 # via onadata dnspython==2.6.1 # via pymongo -docutils==0.19 +docutils==0.20.1 # via sphinx dpath==2.1.6 # via onadata @@ -206,9 +202,7 @@ httplib2==0.22.0 idna==3.6 # via requests ijson==3.2.3 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator imagesize==1.4.1 # via sphinx inflection==0.5.1 @@ -222,9 +216,7 @@ jmespath==1.0.1 # boto3 # botocore jsonlines==4.0.0 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator jsonpickle==3.0.3 # via onadata jsonpointer==2.4 @@ -240,12 +232,10 @@ jwcrypto==1.5.6 kombu==5.3.5 # via celery linear-tsv==1.1.0 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator lxml==5.1.0 # via onadata -markdown==3.5.2 +markdown==3.6 # via onadata markupsafe==2.1.5 # via jinja2 @@ -266,7 +256,6 @@ openpyxl==3.0.9 # dataflows-tabulator # onadata # pyxform - # tabulator packaging==24.0 # via sphinx paho-mqtt==2.0.0 @@ -289,7 +278,7 @@ pycparser==2.21 # via cffi pygments==2.17.2 # via sphinx -pyjwt[crypto]==2.8.0 +pyjwt==2.8.0 # via # ona-oidc # onadata @@ -311,9 +300,7 @@ python-memcached==1.62 # via onadata pytz==2024.1 # via - # django # django-query-builder - # djangorestframework # fleming # onadata pyxform==1.12.2 @@ -324,7 +311,7 @@ recaptcha-client==1.0.6 # via onadata redis==5.0.3 # via django-redis -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -340,7 +327,6 @@ requests==2.31.0 # requests-oauthlib # sphinx # tableschema - # tabulator requests-oauthlib==1.4.0 # via google-auth-oauthlib rfc3986==2.0.0 @@ -351,9 +337,9 @@ rpds-py==0.18.0 # referencing rsa==4.9 # via google-auth -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 -sentry-sdk==1.41.0 +sentry-sdk==1.42.0 # via onadata simplejson==3.19.2 # via onadata @@ -367,38 +353,32 @@ six==1.16.0 # linear-tsv # python-dateutil # tableschema - # tabulator snowballstemmer==2.2.0 # via sphinx -sphinx==6.2.1 +sphinx==7.2.6 # via onadata -sphinxcontrib-applehelp==1.0.5 +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.3 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.2 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.4 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.6 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy==2.0.28 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator sqlparse==0.4.4 # via # django # django-debug-toolbar -tableschema==1.20.7 +tableschema==1.20.9 # via datapackage -tabulator==1.53.5 - # via tableschema typing-extensions==4.10.0 # via - # asgiref # jwcrypto # sqlalchemy tzdata==2024.1 @@ -411,8 +391,7 @@ unicodecsv==0.14.1 # datapackage # onadata # tableschema - # tabulator -urllib3==2.0.7 +urllib3==2.2.1 # via # botocore # requests @@ -432,7 +411,6 @@ xlrd==2.0.1 # via # dataflows-tabulator # pyxform - # tabulator xlwt==1.3.0 # via onadata xmltodict==0.13.0 diff --git a/requirements/dev.in b/requirements/dev.in index f9bc526712..767475d4eb 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,15 +2,16 @@ -r base.in django-extensions +flake8 +flaky +httmock ipdb -pylint -pylint-django -yapf isort -prospector -httmock mock +pre-commit +prospector +pylint +pylint-django requests_mock tblib -flake8 -flaky +yapf diff --git a/requirements/dev.pip b/requirements/dev.pip index cb50c8ccdb..fc14b8dfad 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -1,8 +1,8 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/dev.pip requirements/dev.in +# pip-compile --output-file=requirements/dev.pip --strip-extras requirements/dev.in # -e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest # via -r requirements/base.in @@ -38,8 +38,6 @@ astroid==2.15.8 # requirements-detector asttokens==2.4.1 # via stack-data -async-timeout==4.0.3 - # via redis attrs==23.2.0 # via # jsonlines @@ -51,11 +49,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.61 - # via - # dataflows-tabulator - # tabulator -botocore==1.34.61 +boto3==1.34.66 + # via dataflows-tabulator +botocore==1.34.66 # via # boto3 # s3transfer @@ -71,11 +67,12 @@ certifi==2024.2.2 # sentry-sdk cffi==1.16.0 # via cryptography +cfgv==3.4.0 + # via pre-commit chardet==5.2.0 # via # dataflows-tabulator # datapackage - # tabulator charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -87,7 +84,6 @@ click==8.1.7 # dataflows-tabulator # datapackage # tableschema - # tabulator click-didyoumean==0.3.0 # via celery click-plugins==1.1.1 @@ -99,8 +95,10 @@ cryptography==42.0.5 # jwcrypto # onadata # pyjwt -dataflows-tabulator==1.54.0 - # via datapackage +dataflows-tabulator==1.54.1 + # via + # datapackage + # tableschema datapackage==1.15.4 # via pyfloip decorator==5.1.1 @@ -118,7 +116,9 @@ dict2xml==1.7.5 # via onadata dill==0.3.8 # via pylint -django==3.2.25 +distlib==0.3.8 + # via virtualenv +django==4.0 # via # django-activity-stream # django-cors-headers @@ -156,7 +156,7 @@ django-guardian==2.4.0 # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==2.3.0 +django-oauth-toolkit==2.1.0 # via onadata django-ordered-model==3.7.4 # via onadata @@ -174,7 +174,7 @@ django-taggit==4.0.0 # via onadata django-templated-email==3.0.1 # via onadata -djangorestframework==3.14.0 +djangorestframework==3.15.0 # via # djangorestframework-csv # djangorestframework-gis @@ -196,7 +196,7 @@ djangorestframework-xml==2.0.0 # via onadata dnspython==2.6.1 # via pymongo -docutils==0.19 +docutils==0.20.1 # via sphinx dodgy==0.2.1 # via prospector @@ -206,10 +206,10 @@ elaphe3==0.2.0 # via onadata et-xmlfile==1.1.0 # via openpyxl -exceptiongroup==1.2.0 - # via ipython executing==2.0.1 # via stack-data +filelock==3.13.1 + # via virtualenv flake8==3.8.4 # via # -r requirements/dev.in @@ -236,12 +236,12 @@ httmock==1.4.0 # via -r requirements/dev.in httplib2==0.22.0 # via onadata +identify==2.5.35 + # via pre-commit idna==3.6 # via requests ijson==3.2.3 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator imagesize==1.4.1 # via sphinx importlib-metadata==7.0.2 @@ -267,9 +267,7 @@ jmespath==1.0.1 # boto3 # botocore jsonlines==4.0.0 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator jsonpickle==3.0.3 # via onadata jsonpointer==2.4 @@ -287,12 +285,10 @@ kombu==5.3.5 lazy-object-proxy==1.10.0 # via astroid linear-tsv==1.1.0 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator lxml==5.1.0 # via onadata -markdown==3.5.2 +markdown==3.6 # via onadata markupsafe==2.1.5 # via jinja2 @@ -303,12 +299,14 @@ mccabe==0.6.1 # flake8 # prospector # pylint -mock==4.0.3 +mock==5.1.0 # via -r requirements/dev.in modilabs-python-utils==0.1.5 # via onadata monotonic==1.6 # via analytics-python +nodeenv==1.8.0 + # via pre-commit nose==1.3.7 # via django-nose numpy==1.26.4 @@ -322,10 +320,8 @@ openpyxl==3.0.9 # dataflows-tabulator # onadata # pyxform - # tabulator packaging==24.0 # via - # prospector # requirements-detector # sphinx paho-mqtt==2.0.0 @@ -343,7 +339,10 @@ pillow==10.2.0 platformdirs==4.2.0 # via # pylint + # virtualenv # yapf +pre-commit==3.6.2 + # via -r requirements/dev.in prompt-toolkit==3.0.43 # via # click-repl @@ -378,7 +377,7 @@ pygments==2.17.2 # via # ipython # sphinx -pyjwt[crypto]==2.8.0 +pyjwt==2.8.0 # via # ona-oidc # onadata @@ -422,9 +421,7 @@ python-memcached==1.62 # via onadata pytz==2024.1 # via - # django # django-query-builder - # djangorestframework # fleming # onadata pyxform==1.12.2 @@ -432,12 +429,14 @@ pyxform==1.12.2 # onadata # pyfloip pyyaml==6.0.1 - # via prospector + # via + # pre-commit + # prospector recaptcha-client==1.0.6 # via onadata redis==5.0.3 # via django-redis -referencing==0.33.0 +referencing==0.34.0 # via # jsonschema # jsonschema-specifications @@ -455,7 +454,6 @@ requests==2.31.0 # requests-oauthlib # sphinx # tableschema - # tabulator requests-mock==1.11.0 # via -r requirements/dev.in requests-oauthlib==1.4.0 @@ -470,11 +468,11 @@ rpds-py==0.18.0 # referencing rsa==4.9 # via google-auth -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 semver==3.0.2 # via requirements-detector -sentry-sdk==1.41.0 +sentry-sdk==1.42.0 # via onadata setoptconf==0.3.0 # via prospector @@ -492,48 +490,40 @@ six==1.16.0 # python-dateutil # requests-mock # tableschema - # tabulator snowballstemmer==2.2.0 # via # pydocstyle # sphinx -sphinx==6.2.1 +sphinx==7.2.6 # via onadata -sphinxcontrib-applehelp==1.0.5 +sphinxcontrib-applehelp==1.0.8 # via sphinx -sphinxcontrib-devhelp==1.0.3 +sphinxcontrib-devhelp==1.0.6 # via sphinx -sphinxcontrib-htmlhelp==2.0.2 +sphinxcontrib-htmlhelp==2.0.5 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.4 +sphinxcontrib-qthelp==1.0.7 # via sphinx -sphinxcontrib-serializinghtml==1.1.6 +sphinxcontrib-serializinghtml==1.1.10 # via sphinx sqlalchemy==2.0.28 - # via - # dataflows-tabulator - # tabulator + # via dataflows-tabulator sqlparse==0.4.4 # via # django # django-debug-toolbar stack-data==0.6.3 # via ipython -tableschema==1.20.7 +tableschema==1.20.9 # via datapackage -tabulator==1.53.5 - # via tableschema tblib==3.0.0 # via -r requirements/dev.in toml==0.10.2 # via requirements-detector tomli==2.0.1 - # via - # ipdb - # pylint - # yapf + # via yapf tomlkit==0.12.4 # via pylint traitlets==5.14.2 @@ -542,8 +532,6 @@ traitlets==5.14.2 # matplotlib-inline typing-extensions==4.10.0 # via - # asgiref - # astroid # jwcrypto # sqlalchemy tzdata==2024.1 @@ -556,8 +544,7 @@ unicodecsv==0.14.1 # datapackage # onadata # tableschema - # tabulator -urllib3==2.0.7 +urllib3==2.2.1 # via # botocore # requests @@ -569,6 +556,8 @@ vine==5.1.0 # amqp # celery # kombu +virtualenv==20.25.1 + # via pre-commit wcwidth==0.2.13 # via prompt-toolkit wrapt==1.16.0 @@ -579,12 +568,14 @@ xlrd==2.0.1 # via # dataflows-tabulator # pyxform - # tabulator xlwt==1.3.0 # via onadata xmltodict==0.13.0 # via onadata yapf==0.40.2 # via -r requirements/dev.in -zipp==3.18.0 +zipp==3.18.1 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/mysql.pip b/requirements/mysql.pip deleted file mode 100644 index 658a2bfda2..0000000000 --- a/requirements/mysql.pip +++ /dev/null @@ -1 +0,0 @@ -MySQL-python>=1.2.2 \ No newline at end of file diff --git a/requirements/s3.in b/requirements/s3.in index 108712204d..9dcb6b3af8 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,3 +1,3 @@ -django-storages -django >=3.2.25,<4 boto3 +django ==4.0,<5 +django-storages diff --git a/requirements/s3.pip b/requirements/s3.pip index d03912b6b1..8e68f1a5fa 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -1,18 +1,18 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/s3.pip requirements/s3.in +# pip-compile --output-file=requirements/s3.pip --strip-extras requirements/s3.in # asgiref==3.7.2 # via django -boto3==1.34.61 +boto3==1.34.66 # via -r requirements/s3.in -botocore==1.34.61 +botocore==1.34.66 # via # boto3 # s3transfer -django==3.2.25 +django==4.0 # via # -r requirements/s3.in # django-storages @@ -24,15 +24,11 @@ jmespath==1.0.1 # botocore python-dateutil==2.9.0.post0 # via botocore -pytz==2024.1 - # via django -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 six==1.16.0 # via python-dateutil sqlparse==0.4.4 # via django -typing-extensions==4.10.0 - # via asgiref -urllib3==2.0.7 +urllib3==2.2.1 # via botocore diff --git a/requirements/ses.in b/requirements/ses.in index 4825ec8368..d0a01aa09d 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,3 +1,3 @@ boto -django >=3.2.25,<4 +django ==4.0,<5 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index aee2de5254..5162be5bb4 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/ses.pip requirements/ses.in +# pip-compile --output-file=requirements/ses.pip --strip-extras requirements/ses.in # asgiref==3.7.2 # via django boto==2.49.0 # via -r requirements/ses.in -boto3==1.34.61 +boto3==1.34.66 # via django-ses -botocore==1.34.61 +botocore==1.34.66 # via # boto3 # s3transfer -django==3.2.25 +django==4.0 # via # -r requirements/ses.in # django-ses @@ -27,16 +27,12 @@ jmespath==1.0.1 python-dateutil==2.9.0.post0 # via botocore pytz==2024.1 - # via - # django - # django-ses -s3transfer==0.10.0 + # via django-ses +s3transfer==0.10.1 # via boto3 six==1.16.0 # via python-dateutil sqlparse==0.4.4 # via django -typing-extensions==4.10.0 - # via asgiref -urllib3==2.0.7 +urllib3==2.2.1 # via botocore diff --git a/setup.cfg b/setup.cfg index 70a7bc93e6..a32edc43a3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,7 +27,7 @@ tests_require = mock requests-mock install_requires = - Django>=3.2.25,<4 + Django==4.0,<5 django-guardian django-registration-redux django-templated-email @@ -68,7 +68,7 @@ install_requires = dict2xml lxml>=4.9.1 #pyxform - pyxform + pyxform==1.12.2 #memcached support pylibmc python-memcached From 72f2a280b2fb7b0178d0eb435dd08e62cc346c75 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Mar 2024 13:47:31 +0300 Subject: [PATCH 02/37] Fix DeprecationWarning: invalid escape sequence Declare the regex pattern as a raw string. --- onadata/apps/logger/models/xform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index c0743e3222..dd3076718a 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -315,11 +315,11 @@ def set_uuid_in_xml(self, file_name=None): # http://ronrothman.com/public/leftbraned/xml-dom-minidom-toprettyxml-\ # and-silly-whitespace/ text_re = re.compile(r"(>)\n\s*(\s[^<>\s].*?)\n\s*(\s)\n( )*") + output_re = re.compile(r"\n.*()\n( )*") pretty_xml = text_re.sub( lambda m: "".join(m.group(1, 2, 3)), self.xml.decode("utf-8") ) - inline_output = output_re.sub("\g<1>", pretty_xml) # noqa + inline_output = output_re.sub(r"\g<1>", pretty_xml) # noqa inline_output = re.compile(r"").sub( "", inline_output ) From 4f7aa4b4fbbf8064d8722af5a5a107b9413f3422 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Mar 2024 15:56:05 +0300 Subject: [PATCH 03/37] Django 4: remove pytz --- onadata/apps/logger/models/instance.py | 12 +++------- onadata/apps/logger/models/xform.py | 8 +------ .../apps/logger/tests/models/test_instance.py | 24 +++++++++---------- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index dfe6337b55..81201f160f 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -20,7 +20,6 @@ from django.utils import timezone from django.utils.translation import gettext as _ -import pytz from deprecated import deprecated from taggit.managers import TaggableManager @@ -188,12 +187,7 @@ def _update_submission_count_for_today( form_id: int, incr: bool = True, date_created=None ): # Track submissions made today - current_timzone_name = timezone.get_current_timezone_name() - current_timezone = pytz.timezone(current_timzone_name) - today = datetime.today() - current_date = current_timezone.localize( - datetime(today.year, today.month, today.day) - ).isoformat() + current_date = timezone.localtime().isoformat() date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}" count_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}" @@ -201,8 +195,8 @@ def _update_submission_count_for_today( cache.set(date_cache_key, current_date, 86400) if date_created: - date_created = current_timezone.localize( - datetime(date_created.year, date_created.month, date_created.day) + date_created = date_created.astimezone( + timezone.get_current_timezone() ).isoformat() current_count = cache.get(count_cache_key) diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index dd3076718a..195b304de5 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -24,7 +24,6 @@ from django.utils.translation import gettext as _ from django.utils.translation import gettext_lazy -import pytz from guardian.models import GroupObjectPermissionBase, UserObjectPermissionBase from pyxform import SurveyElementBuilder, constants, create_survey_element_from_dict from pyxform.question import Question @@ -1159,12 +1158,7 @@ def submission_count(self, force_update=False): @property def submission_count_for_today(self): """Returns the submissions count for the current day.""" - current_timzone_name = timezone.get_current_timezone_name() - current_timezone = pytz.timezone(current_timzone_name) - today = datetime.today() - current_date = current_timezone.localize( - datetime(today.year, today.month, today.day) - ).isoformat() + current_date = timezone.localtime().isoformat() count = ( cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") if cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}") diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 09cef2fcb9..5dcf01d63c 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -1,15 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Test Instance model. +""" import os -import pytz -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta, timezone from django.http.request import HttpRequest +from django.test import override_settings from django.utils.timezone import utc + from django_digest.test import DigestAuth -from django.test import override_settings -from mock import patch, Mock +from mock import Mock, patch -from onadata.apps.logger.models import XForm, Instance, SubmissionReview +from onadata.apps.logger.models import Instance, SubmissionReview, XForm from onadata.apps.logger.models.instance import ( get_id_string_from_xml_str, numeric_checker, @@ -23,10 +26,7 @@ from onadata.libs.serializers.submission_review_serializer import ( SubmissionReviewSerializer, ) -from onadata.libs.utils.common_tags import ( - MONGO_STRFTIME, - SUBMITTED_BY, -) +from onadata.libs.utils.common_tags import MONGO_STRFTIME, SUBMITTED_BY class TestInstance(TestBase): @@ -67,7 +67,7 @@ def test_stores_json(self): def test_updates_json_date_modified_on_save(self): """_date_modified in `json` field is updated on save""" - old_mocked_now = datetime(2023, 9, 21, 8, 27, 0, tzinfo=pytz.utc) + old_mocked_now = datetime(2023, 9, 21, 8, 27, 0, tzinfo=timezone.utc) with patch("django.utils.timezone.now", Mock(return_value=old_mocked_now)): self._publish_transportation_form_and_submit_instance() @@ -79,7 +79,7 @@ def test_updates_json_date_modified_on_save(self): ) # After saving the date_modified in json should update - mocked_now = datetime(2023, 9, 21, 9, 3, 0, tzinfo=pytz.utc) + mocked_now = datetime(2023, 9, 21, 9, 3, 0, tzinfo=timezone.utc) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): instance.save() From 4e7a9b176568a3bd9c66c886039da5e214ecee61 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 20 Mar 2024 16:00:12 +0300 Subject: [PATCH 04/37] Django 4: update azure requirement packages --- requirements/azure.pip | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/requirements/azure.pip b/requirements/azure.pip index 8c8828a584..91dfe77194 100644 --- a/requirements/azure.pip +++ b/requirements/azure.pip @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --output-file=requirements/azure.pip requirements/azure.in +# pip-compile --output-file=requirements/azure.pip --strip-extras requirements/azure.in # asgiref==3.7.2 # via django @@ -22,11 +22,11 @@ cryptography==42.0.5 # via # -r requirements/azure.in # azure-storage-blob -django==3.2.25 +django==4.0 # via # -r requirements/azure.in # django-storages -django-storages[azure]==1.14.2 +django-storages==1.14.2 # via -r requirements/azure.in idna==3.6 # via requests @@ -34,8 +34,6 @@ isodate==0.6.1 # via azure-storage-blob pycparser==2.21 # via cffi -pytz==2024.1 - # via django requests==2.31.0 # via azure-core six==1.16.0 From 95a10704c75524a2cec091673211f428694ef5d9 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 15:41:14 +0300 Subject: [PATCH 05/37] Django 4: remove deprecated pytz --- .../api/tests/viewsets/test_data_viewset.py | 19 ++++---- .../tests/viewsets/test_project_viewset.py | 26 +++++------ .../api/tests/viewsets/test_xform_viewset.py | 7 ++- onadata/apps/api/viewsets/xform_viewset.py | 7 +-- onadata/apps/logger/models/instance.py | 7 ++- .../apps/logger/tests/models/test_instance.py | 6 +-- .../tests/models/test_project_invitation.py | 22 +++++----- onadata/apps/logger/views.py | 19 ++++---- onadata/apps/main/tests/test_process.py | 8 ++-- onadata/apps/main/tests/test_signals.py | 10 ++--- onadata/apps/viewer/views.py | 14 +++--- onadata/libs/mixins/openrosa_headers_mixin.py | 7 +-- onadata/libs/renderers/renderers.py | 7 +-- onadata/libs/utils/logger_tools.py | 4 +- requirements/base.in | 2 +- requirements/base.pip | 13 +++--- requirements/dev.in | 3 +- requirements/dev.pip | 44 +++++++++++-------- setup.cfg | 1 - 19 files changed, 107 insertions(+), 119 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index bef0b1a0bf..f33dc18ee1 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -4,15 +4,14 @@ """ from __future__ import unicode_literals +import csv import datetime import json import logging import os -import pytz -import csv -from io import StringIO from builtins import open from datetime import timedelta +from io import StringIO from tempfile import NamedTemporaryFile from django.conf import settings @@ -30,7 +29,7 @@ from django_digest.test import DigestAuth from flaky import flaky from httmock import HTTMock, urlmatch -from mock import patch, Mock +from mock import Mock, patch from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( TestAbstractViewSet, @@ -2374,7 +2373,7 @@ def test_geojson_format(self): def test_geotraces_in_repeats(self): # publish sample geotrace submissions md = """ - | survey | + | survey | | | type | name | label | required | calculation | | | begin repeat | segment | Waterway trace | | | | | calculate | point_position | | | position(..)| @@ -2436,7 +2435,7 @@ def test_geotraces_in_repeats(self): def test_geoshapes_in_repeats(self): # publish sample geoshape submissions md = """ - | survey | + | survey | | | type | name | label | required | calculation | | | begin repeat | segment | Waterway trace | | | | | calculate | point_position | | | position(..)| @@ -2507,7 +2506,7 @@ def test_geoshapes_in_repeats(self): def test_empty_geotraces_in_repeats(self): # publish sample geotrace submissions md = """ - | survey | + | survey | | | type | name | label | required | calculation | | | begin repeat | segment | Waterway trace | | | | | calculate | point_position | | | position(..)| @@ -2551,7 +2550,7 @@ def test_empty_geotraces_in_repeats(self): [36.805943, -1.268118], [36.808822, -1.269405], ], - }, + }, "properties": {"id": instances[1].pk, "xform": self.xform.pk}, }, ], @@ -2561,7 +2560,7 @@ def test_empty_geotraces_in_repeats(self): def test_empty_geoshapes_in_repeats(self): # publish sample geoshape submissions md = """ - | survey | + | survey | | | type | name | label | required | calculation | | | begin repeat | segment | Waterway trace | | | | | calculate | point_position | | | position(..)| @@ -3518,7 +3517,7 @@ def test_data_list_xml_format(self): """Test DataViewSet list XML""" # create submission media_file = "1335783522563.jpg" - mocked_now = datetime.datetime(2023, 9, 20, 12, 49, 0, tzinfo=pytz.utc) + mocked_now = datetime.datetime(2023, 9, 20, 12, 49, 0, tzinfo=timezone.utc) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): self._make_submission_w_attachment( diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index f7b847aa4a..2a787b6f6c 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -3,24 +3,24 @@ Test ProjectViewSet module. """ import json -import pytz import os - from collections import OrderedDict -from six import iteritems -from operator import itemgetter from datetime import datetime +from operator import itemgetter from django.conf import settings -from django.db.models import Q +from django.contrib.auth import get_user_model from django.core.cache import cache +from django.db.models import Q from django.test import override_settings -from django.contrib.auth import get_user_model -from rest_framework.authtoken.models import Token -from httmock import HTTMock, urlmatch -from mock import MagicMock, patch, Mock +from django.utils import timezone + import dateutil.parser import requests +from httmock import HTTMock, urlmatch +from mock import MagicMock, Mock, patch +from rest_framework.authtoken.models import Token +from six import iteritems from onadata.apps.api import tools from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( @@ -34,11 +34,10 @@ from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.team_viewset import TeamViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.logger.models import Project, XForm, XFormVersion, ProjectInvitation +from onadata.apps.logger.models import Project, ProjectInvitation, XForm, XFormVersion from onadata.apps.main.models import MetaData from onadata.libs import permissions as role from onadata.libs.models.share_project import ShareProject -from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key from onadata.libs.permissions import ( ROLES_ORDERED, DataEntryMinorRole, @@ -55,6 +54,7 @@ BaseProjectSerializer, ProjectSerializer, ) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key User = get_user_model() @@ -3320,7 +3320,7 @@ def test_revoke_invite(self): ) post_data = {"invitation_id": invitation.pk} request = self.factory.post("/", data=post_data, **self.extra) - mocked_now = datetime(2023, 5, 25, 10, 51, 0, tzinfo=pytz.utc) + mocked_now = datetime(2023, 5, 25, 10, 51, 0, tzinfo=timezone.utc) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): response = self.view(request, pk=self.project.pk) @@ -3423,7 +3423,7 @@ def test_resend_invite(self, mock_send_mail): email="jandoe@example.com", role="editor" ) post_data = {"invitation_id": invitation.pk} - mocked_now = datetime(2023, 5, 25, 10, 51, 0, tzinfo=pytz.utc) + mocked_now = datetime(2023, 5, 25, 10, 51, 0, tzinfo=timezone.utc) request = self.factory.post("/", data=post_data, **self.extra) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index ae55a4d1bc..049a0b12cd 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -8,7 +8,6 @@ import csv import json import os -import pytz import re from builtins import open from collections import OrderedDict @@ -33,7 +32,6 @@ from flaky import flaky from httmock import HTTMock from mock import Mock, patch -from onadata.libs.utils.api_export_tools import get_existing_file_format from rest_framework import status from onadata.apps.api.tests.mocked_data import ( @@ -58,8 +56,8 @@ from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment, Instance, Project, XForm from onadata.apps.logger.models.xform_version import XFormVersion -from onadata.apps.logger.xform_instance_parser import XLSFormError from onadata.apps.logger.views import delete_xform +from onadata.apps.logger.xform_instance_parser import XLSFormError from onadata.apps.main.models import MetaData from onadata.apps.messaging.constants import FORM_UPDATED, XFORM from onadata.apps.viewer.models import Export @@ -78,6 +76,7 @@ XFormBaseSerializer, XFormSerializer, ) +from onadata.libs.utils.api_export_tools import get_existing_file_format from onadata.libs.utils.cache_tools import ( ENKETO_URL_CACHE, PROJ_FORMS_CACHE, @@ -3840,7 +3839,7 @@ def test_csv_export_with_win_excel_utf8(self): self._publish_xls_form_to_project(xlsform_path=xlsform_path) # submit one hxl instance _submission_time = parse_datetime("2013-02-18 15:54:01Z") - mock_date_modified = datetime(2023, 9, 20, 11, 41, 0, tzinfo=pytz.utc) + mock_date_modified = datetime(2023, 9, 20, 11, 41, 0, tzinfo=utc) with patch( "django.utils.timezone.now", Mock(return_value=mock_date_modified) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index 749b4a74d4..fb79f2ad89 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -76,17 +76,14 @@ XFormVersionListSerializer, ) from onadata.libs.utils.api_export_tools import ( + _get_export_type, custom_response_handler, get_async_response, get_existing_file_format, process_async_export, response_for_format, - _get_export_type, -) -from onadata.libs.utils.cache_tools import ( - PROJ_OWNER_CACHE, - safe_delete, ) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_delete from onadata.libs.utils.common_tools import json_stream from onadata.libs.utils.csv_import import ( get_async_csv_submission_status, diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 81201f160f..5ee33bc8be 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -6,7 +6,6 @@ import sys from datetime import datetime -from celery import current_task from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.gis.db import models @@ -20,6 +19,7 @@ from django.utils import timezone from django.utils.translation import gettext as _ +from celery import current_task from deprecated import deprecated from taggit.managers import TaggableManager @@ -32,8 +32,6 @@ get_uuid_from_xml, ) from onadata.celeryapp import app -from onadata.libs.utils.common_tools import report_exception -from onadata.libs.utils.model_tools import queryset_iterator from onadata.libs.data.query import get_numeric_fields from onadata.libs.utils.cache_tools import ( DATAVIEW_COUNT, @@ -74,8 +72,9 @@ XFORM_ID, XFORM_ID_STRING, ) +from onadata.libs.utils.common_tools import report_exception from onadata.libs.utils.dict_tools import get_values_matching_key -from onadata.libs.utils.model_tools import set_uuid +from onadata.libs.utils.model_tools import queryset_iterator, set_uuid from onadata.libs.utils.timing import calculate_duration # pylint: disable=invalid-name diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 5dcf01d63c..89222d97bc 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -3,7 +3,7 @@ Test Instance model. """ import os -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta from django.http.request import HttpRequest from django.test import override_settings @@ -67,7 +67,7 @@ def test_stores_json(self): def test_updates_json_date_modified_on_save(self): """_date_modified in `json` field is updated on save""" - old_mocked_now = datetime(2023, 9, 21, 8, 27, 0, tzinfo=timezone.utc) + old_mocked_now = datetime(2023, 9, 21, 8, 27, 0, tzinfo=utc) with patch("django.utils.timezone.now", Mock(return_value=old_mocked_now)): self._publish_transportation_form_and_submit_instance() @@ -79,7 +79,7 @@ def test_updates_json_date_modified_on_save(self): ) # After saving the date_modified in json should update - mocked_now = datetime(2023, 9, 21, 9, 3, 0, tzinfo=timezone.utc) + mocked_now = datetime(2023, 9, 21, 9, 3, 0, tzinfo=utc) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): instance.save() diff --git a/onadata/apps/logger/tests/models/test_project_invitation.py b/onadata/apps/logger/tests/models/test_project_invitation.py index 0c0646c1e1..dcca3590cf 100644 --- a/onadata/apps/logger/tests/models/test_project_invitation.py +++ b/onadata/apps/logger/tests/models/test_project_invitation.py @@ -2,8 +2,10 @@ Tests for ProjectInvitation model """ from datetime import datetime -from unittest.mock import patch, Mock -import pytz +from unittest.mock import Mock, patch + +from django.utils import timezone + from onadata.apps.logger.models import ProjectInvitation from onadata.apps.main.tests.test_base import TestBase from onadata.libs.utils.user_auth import get_user_default_project @@ -20,10 +22,10 @@ def setUp(self) -> None: def test_creation(self): """We can create a ProjectInvitation object""" - created_at = datetime(2023, 5, 17, 14, 21, 0, tzinfo=pytz.utc) - resent_at = datetime(2023, 5, 17, 14, 24, 0, tzinfo=pytz.utc) - accepted_at = datetime(2023, 5, 17, 14, 25, 0, tzinfo=pytz.utc) - revoked_at = datetime(2023, 5, 17, 14, 26, 0, tzinfo=pytz.utc) + created_at = datetime(2023, 5, 17, 14, 21, 0, tzinfo=timezone.utc) + resent_at = datetime(2023, 5, 17, 14, 24, 0, tzinfo=timezone.utc) + accepted_at = datetime(2023, 5, 17, 14, 25, 0, tzinfo=timezone.utc) + revoked_at = datetime(2023, 5, 17, 14, 26, 0, tzinfo=timezone.utc) jane = self._create_user("jane", "1234") with patch("django.utils.timezone.now", Mock(return_value=created_at)): @@ -67,7 +69,7 @@ def test_defaults(self): def test_revoke(self): """Calling revoke method works correctly""" - mocked_now = datetime(2023, 5, 25, 11, 17, 0, tzinfo=pytz.utc) + mocked_now = datetime(2023, 5, 25, 11, 17, 0, tzinfo=timezone.utc) with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): invitation = ProjectInvitation.objects.create( @@ -82,7 +84,7 @@ def test_revoke(self): self.assertEqual(invitation.status, ProjectInvitation.Status.REVOKED) # setting revoked_at explicitly works - revoked_at = datetime(2023, 5, 10, 11, 17, 0, tzinfo=pytz.utc) + revoked_at = datetime(2023, 5, 10, 11, 17, 0, tzinfo=timezone.utc) invitation = ProjectInvitation.objects.create( email="john@example.com", project=self.project, @@ -96,7 +98,7 @@ def test_revoke(self): def test_accept(self): """Calling accept method works correctly""" - mocked_now = datetime(2023, 5, 25, 11, 17, 0, tzinfo=pytz.utc) + mocked_now = datetime(2023, 5, 25, 11, 17, 0, tzinfo=timezone.utc) jane = self._create_user("jane", "1234") with patch("django.utils.timezone.now", Mock(return_value=mocked_now)): @@ -113,7 +115,7 @@ def test_accept(self): self.assertEqual(invitation.status, ProjectInvitation.Status.ACCEPTED) # setting accepted_at explicitly works - accepted_at = datetime(2023, 5, 10, 11, 17, 0, tzinfo=pytz.utc) + accepted_at = datetime(2023, 5, 10, 11, 17, 0, tzinfo=timezone.utc) invitation = ProjectInvitation.objects.create( email="john@example.com", project=self.project, diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 21f75b2d69..1308b8a900 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -6,12 +6,10 @@ import tempfile from datetime import datetime -import pytz -import six from django.conf import settings from django.contrib import messages -from django.contrib.auth.decorators import login_required from django.contrib.auth import get_user_model +from django.contrib.auth.decorators import login_required from django.core.files import File from django.core.files.storage import get_storage_class from django.http import ( @@ -24,21 +22,24 @@ from django.shortcuts import get_object_or_404, render from django.template import RequestContext, loader from django.urls import reverse +from django.utils import timezone from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from django.views.decorators.http import require_GET, require_http_methods, require_POST + +import six from django_digest import HttpDigestAuthenticator +from onadata.apps.api.tools import get_host_domain from onadata.apps.logger.import_tools import import_instances_from_zip from onadata.apps.logger.models.attachment import Attachment from onadata.apps.logger.models.instance import Instance from onadata.apps.logger.models.xform import XForm -from onadata.apps.api.tools import get_host_domain from onadata.apps.main.models import MetaData, UserProfile from onadata.libs.exceptions import EnketoError +from onadata.libs.utils.cache_tools import USER_PROFILE_PREFIX, cache from onadata.libs.utils.decorators import is_owner from onadata.libs.utils.log import Actions, audit_log -from onadata.libs.utils.cache_tools import cache, USER_PROFILE_PREFIX from onadata.libs.utils.logger_tools import ( BaseOpenRosaResponse, OpenRosaResponse, @@ -250,9 +251,7 @@ def formList(request, username): # noqa N802 request, "xformsList.xml", data, content_type="text/xml; charset=utf-8" ) response["X-OpenRosa-Version"] = "1.0" - response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( - "%a, %d %b %Y %H:%M:%S %Z" - ) + response["Date"] = timezone.localtime().strftime("%a, %d %b %Y %H:%M:%S %Z") return response @@ -288,9 +287,7 @@ def xformsManifest(request, username, id_string): # noqa N802 content_type="text/xml; charset=utf-8", ) response["X-OpenRosa-Version"] = "1.0" - response["Date"] = datetime.now(pytz.timezone(settings.TIME_ZONE)).strftime( - "%a, %d %b %Y %H:%M:%S %Z" - ) + response["Date"] = timezone.localtime().strftime("%a, %d %b %Y %H:%M:%S %Z") return response diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 27b6848e77..9af3918aa4 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -16,9 +16,9 @@ from django.core.files.uploadedfile import UploadedFile from django.test.testcases import SerializeMixin from django.urls import reverse +from django.utils import timezone import openpyxl -import pytz import requests from django_digest.test import Client as DigestClient from flaky import flaky @@ -77,9 +77,9 @@ def _update_dynamic_data(self): """ for uuid, submission_time in iteritems(self.uuid_to_submission_times): i = self.xform.instances.get(uuid=uuid) - i.date_created = pytz.timezone("UTC").localize( - datetime.strptime(submission_time, MONGO_STRFTIME) - ) + i.date_created = datetime.strptime( + submission_time, MONGO_STRFTIME + ).astimezone(timezone.utc) i.json = i.get_full_dict() i.save() diff --git a/onadata/apps/main/tests/test_signals.py b/onadata/apps/main/tests/test_signals.py index 0484910d29..fe9950c094 100644 --- a/onadata/apps/main/tests/test_signals.py +++ b/onadata/apps/main/tests/test_signals.py @@ -1,17 +1,15 @@ - """Tests for onadata.apps.main.signals module""" from datetime import datetime from unittest.mock import Mock, patch -import pytz - from django.contrib.auth import get_user_model +from django.utils import timezone +from onadata.apps.logger.models import Project, ProjectInvitation from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.logger.models import ProjectInvitation, Project -from onadata.libs.utils.user_auth import get_user_default_project from onadata.libs.permissions import EditorRole, ManagerRole +from onadata.libs.utils.user_auth import get_user_default_project User = get_user_model() @@ -27,7 +25,7 @@ def setUp(self): project=self.project, role="editor", ) - self.mocked_now = datetime(2023, 6, 21, 14, 29, 0, tzinfo=pytz.utc) + self.mocked_now = datetime(2023, 6, 21, 14, 29, 0, tzinfo=timezone.utc) def test_accept_invitation(self): """Accept invitation works""" diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index ff686bdbe7..7ae8f6b66f 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -24,10 +24,10 @@ from django.shortcuts import get_object_or_404, redirect, render from django.template import loader from django.urls import reverse +from django.utils import timezone from django.utils.translation import gettext as _ from django.views.decorators.http import require_POST -import pytz import requests from dict2xml import dict2xml from dpath import util as dpath_util @@ -56,7 +56,7 @@ str_to_bool, ) from onadata.libs.utils.google import create_flow -from onadata.libs.utils.image_tools import image_url, generate_media_download_url +from onadata.libs.utils.image_tools import generate_media_download_url, image_url from onadata.libs.utils.log import Actions, audit_log from onadata.libs.utils.logger_tools import ( generate_content_disposition_header, @@ -82,12 +82,12 @@ def _get_start_end_submission_time(request): end = None try: if request.GET.get("start"): - start = pytz.timezone("UTC").localize( - datetime.strptime(request.GET["start"], "%y_%m_%d_%H_%M_%S") - ) + start = datetime.strptime( + request.GET["start"], "%y_%m_%d_%H_%M_%S" + ).astimezone(timezone.utc) if request.GET.get("end"): - end = pytz.timezone("UTC").localize( - datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S") + end = datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S").astimezone( + timezone.utc ) except ValueError: return HttpResponseBadRequest( diff --git a/onadata/libs/mixins/openrosa_headers_mixin.py b/onadata/libs/mixins/openrosa_headers_mixin.py index fd61389a9f..29ef886ca1 100644 --- a/onadata/libs/mixins/openrosa_headers_mixin.py +++ b/onadata/libs/mixins/openrosa_headers_mixin.py @@ -2,11 +2,8 @@ """ OpenRosaHeadersMixin module """ -from datetime import datetime - from django.conf import settings - -import pytz +from django.utils import timezone # 10,000,000 bytes DEFAULT_CONTENT_LENGTH = getattr(settings, "DEFAULT_CONTENT_LENGTH", 10000000) @@ -17,7 +14,7 @@ def get_openrosa_headers(request, location=True): Returns a dict with OpenRosa headers 'Date', 'X-OpenRosa-Version', 'X-OpenRosa-Accept-Content-Length' and 'Location'. """ - now = datetime.now(pytz.timezone(settings.TIME_ZONE)) + now = timezone.localtime() data = { "Date": now.strftime("%a, %d %b %Y %H:%M:%S %Z"), "X-OpenRosa-Version": "1.0", diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 4410db62c2..15d5e49756 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -13,7 +13,6 @@ from django.utils.encoding import force_str, smart_str from django.utils.xmlutils import SimplerXMLGenerator -import pytz import six from rest_framework import negotiation from rest_framework.renderers import ( @@ -54,10 +53,8 @@ def floip_rows_list(data): """ try: _submission_time = ( - pytz.timezone("UTC") - .localize(parse_datetime(data["_submission_time"])) - .isoformat() - ) + parse_datetime(data["_submission_time"]).astimezone(timezone.utc) + ).isoformat() except ValueError: _submission_time = data["_submission_time"] diff --git a/onadata/libs/utils/logger_tools.py b/onadata/libs/utils/logger_tools.py index 818bf1be52..412af0ac03 100644 --- a/onadata/libs/utils/logger_tools.py +++ b/onadata/libs/utils/logger_tools.py @@ -37,7 +37,6 @@ from django.utils.encoding import DjangoUnicodeDecodeError from django.utils.translation import gettext as _ -import pytz from defusedxml.ElementTree import ParseError, fromstring from dict2xml import dict2xml from modilabs.utils.subprocess_timeout import ProcessTimedOut @@ -842,8 +841,7 @@ def set_default_openrosa_headers(response): """Sets the default OpenRosa headers into a ``response`` object.""" response["Content-Type"] = "text/html; charset=utf-8" response["X-OpenRosa-Accept-Content-Length"] = DEFAULT_CONTENT_LENGTH - tz = pytz.timezone(settings.TIME_ZONE) - dt = datetime.now(tz).strftime("%a, %d %b %Y %H:%M:%S %Z") + dt = timezone.localtime().strftime("%a, %d %b %Y %H:%M:%S %Z") response["Date"] = dt response[OPEN_ROSA_VERSION_HEADER] = OPEN_ROSA_VERSION response["Content-Type"] = DEFAULT_CONTENT_TYPE diff --git a/requirements/base.in b/requirements/base.in index 3307409f37..b9286956c0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -7,5 +7,5 @@ -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip -e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/ona-oidc.git@v1.0.3#egg=ona-oidc +-e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc -e git+https://github.com/onaio/savreaderwriter.git@fix-pep-440-issues#egg=savreaderwriter diff --git a/requirements/base.pip b/requirements/base.pip index 53b832d678..4284892d08 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -8,7 +8,7 @@ # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@v1.0.3#egg=ona-oidc +-e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in @@ -26,7 +26,7 @@ analytics-python==1.4.post1 # via onadata appoptics-metrics==5.1.0 # via onadata -asgiref==3.7.2 +asgiref==3.8.0 # via # django # django-cors-headers @@ -41,9 +41,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.66 +boto3==1.34.67 # via dataflows-tabulator -botocore==1.34.66 +botocore==1.34.67 # via # boto3 # s3transfer @@ -189,7 +189,7 @@ future==1.0.0 # via python-json2xlsclient geojson==3.1.0 # via onadata -google-auth==2.28.2 +google-auth==2.29.0 # via # google-auth-oauthlib # onadata @@ -302,7 +302,6 @@ pytz==2024.1 # via # django-query-builder # fleming - # onadata pyxform==1.12.2 # via # onadata @@ -339,7 +338,7 @@ rsa==4.9 # via google-auth s3transfer==0.10.1 # via boto3 -sentry-sdk==1.42.0 +sentry-sdk==1.43.0 # via onadata simplejson==3.19.2 # via onadata diff --git a/requirements/dev.in b/requirements/dev.in index 767475d4eb..3762559028 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -2,14 +2,13 @@ -r base.in django-extensions -flake8 flaky httmock ipdb isort mock pre-commit -prospector +prospector>=1.10.3 pylint pylint-django requests_mock diff --git a/requirements/dev.pip b/requirements/dev.pip index fc14b8dfad..d94e34baaa 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -8,7 +8,7 @@ # via -r requirements/base.in -e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@v1.0.3#egg=ona-oidc +-e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc # via -r requirements/base.in -e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip # via -r requirements/base.in @@ -26,7 +26,7 @@ analytics-python==1.4.post1 # via onadata appoptics-metrics==5.1.0 # via onadata -asgiref==3.7.2 +asgiref==3.8.0 # via # django # django-cors-headers @@ -49,9 +49,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.66 +boto3==1.34.67 # via dataflows-tabulator -botocore==1.34.66 +botocore==1.34.67 # via # boto3 # s3transfer @@ -210,10 +210,10 @@ executing==2.0.1 # via stack-data filelock==3.13.1 # via virtualenv -flake8==3.8.4 +flake8==5.0.4 # via - # -r requirements/dev.in # flake8-polyfill + # prospector flake8-polyfill==1.0.2 # via pep8-naming flaky==3.8.1 @@ -224,7 +224,11 @@ future==1.0.0 # via python-json2xlsclient geojson==3.1.0 # via onadata -google-auth==2.28.2 +gitdb==4.0.11 + # via gitpython +gitpython==3.1.42 + # via prospector +google-auth==2.29.0 # via # google-auth-oauthlib # onadata @@ -244,7 +248,7 @@ ijson==3.2.3 # via dataflows-tabulator imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.2 +importlib-metadata==7.1.0 # via yapf inflection==0.5.1 # via djangorestframework-jsonapi @@ -294,7 +298,7 @@ markupsafe==2.1.5 # via jinja2 matplotlib-inline==0.1.6 # via ipython -mccabe==0.6.1 +mccabe==0.7.0 # via # flake8 # prospector @@ -322,6 +326,7 @@ openpyxl==3.0.9 # pyxform packaging==24.0 # via + # prospector # requirements-detector # sphinx paho-mqtt==2.0.0 @@ -347,7 +352,7 @@ prompt-toolkit==3.0.43 # via # click-repl # ipython -prospector==1.4.1.1 +prospector==1.10.3 # via -r requirements/dev.in psycopg2-binary==2.9.9 # via onadata @@ -361,7 +366,7 @@ pyasn1==0.5.1 # rsa pyasn1-modules==0.3.0 # via google-auth -pycodestyle==2.6.0 +pycodestyle==2.9.1 # via # flake8 # prospector @@ -369,7 +374,7 @@ pycparser==2.21 # via cffi pydocstyle==6.3.0 # via prospector -pyflakes==2.2.0 +pyflakes==2.5.0 # via # flake8 # prospector @@ -393,13 +398,13 @@ pylint==2.17.7 # pylint-plugin-utils pylint-celery==0.3 # via prospector -pylint-django==2.1.0 +pylint-django==2.5.3 # via # -r requirements/dev.in # prospector pylint-flask==0.6 # via prospector -pylint-plugin-utils==0.8.2 +pylint-plugin-utils==0.7 # via # prospector # pylint-celery @@ -423,7 +428,6 @@ pytz==2024.1 # via # django-query-builder # fleming - # onadata pyxform==1.12.2 # via # onadata @@ -472,9 +476,9 @@ s3transfer==0.10.1 # via boto3 semver==3.0.2 # via requirements-detector -sentry-sdk==1.42.0 +sentry-sdk==1.43.0 # via onadata -setoptconf==0.3.0 +setoptconf-tmp==0.3.1 # via prospector simplejson==3.19.2 # via onadata @@ -490,6 +494,8 @@ six==1.16.0 # python-dateutil # requests-mock # tableschema +smmap==5.0.1 + # via gitdb snowballstemmer==2.2.0 # via # pydocstyle @@ -521,7 +527,9 @@ tableschema==1.20.9 tblib==3.0.0 # via -r requirements/dev.in toml==0.10.2 - # via requirements-detector + # via + # prospector + # requirements-detector tomli==2.0.1 # via yapf tomlkit==0.12.4 diff --git a/setup.cfg b/setup.cfg index a32edc43a3..a5210b0110 100644 --- a/setup.cfg +++ b/setup.cfg @@ -88,7 +88,6 @@ install_requires = numpy Pillow python-dateutil - pytz requests simplejson uwsgi From b1f35eb1c0ac482d34a8025b1fb1c02666622785 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 16:38:08 +0300 Subject: [PATCH 06/37] pylintrc: address deprecation warning - use full path for Exception class --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 496ac61a1b..76c33d3520 100644 --- a/.pylintrc +++ b/.pylintrc @@ -422,4 +422,4 @@ min-public-methods=2 # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception +overgeneral-exceptions=builtins.Exception From 5aad6b5df3da5e163e9ad561df3d3dfe20429f69 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 16:40:41 +0300 Subject: [PATCH 07/37] Use hiredis for performance with redis-py --- requirements/base.pip | 6 +++++- requirements/dev.pip | 6 +++++- setup.cfg | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index 4284892d08..0cfd03d93e 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -197,6 +197,8 @@ google-auth-oauthlib==1.2.0 # via onadata greenlet==3.0.3 # via sqlalchemy +hiredis==2.3.2 + # via redis httplib2==0.22.0 # via onadata idna==3.6 @@ -309,7 +311,9 @@ pyxform==1.12.2 recaptcha-client==1.0.6 # via onadata redis==5.0.3 - # via django-redis + # via + # django-redis + # onadata referencing==0.34.0 # via # jsonschema diff --git a/requirements/dev.pip b/requirements/dev.pip index d94e34baaa..e3d8a9708c 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -236,6 +236,8 @@ google-auth-oauthlib==1.2.0 # via onadata greenlet==3.0.3 # via sqlalchemy +hiredis==2.3.2 + # via redis httmock==1.4.0 # via -r requirements/dev.in httplib2==0.22.0 @@ -439,7 +441,9 @@ pyyaml==6.0.1 recaptcha-client==1.0.6 # via onadata redis==5.0.3 - # via django-redis + # via + # django-redis + # onadata referencing==0.34.0 # via # jsonschema diff --git a/setup.cfg b/setup.cfg index a5210b0110..bc7909c944 100644 --- a/setup.cfg +++ b/setup.cfg @@ -101,6 +101,7 @@ install_requires = # Deprecation tagging deprecated # Redis cache + redis[hiredis] django-redis # osm defusedxml From a32a41782ba7b0cd23ce509bbaf2379bbca3ddc4 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 17:04:31 +0300 Subject: [PATCH 08/37] Django 4: request.is_ajax() has been removed --- onadata/apps/main/views.py | 6 +++--- onadata/apps/restservice/views.py | 2 +- onadata/apps/viewer/views.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/onadata/apps/main/views.py b/onadata/apps/main/views.py index bf4e82cef5..a2c19d9775 100644 --- a/onadata/apps/main/views.py +++ b/onadata/apps/main/views.py @@ -193,7 +193,7 @@ def set_form(): context = {"message": message, "message_list": message_list} - if request.is_ajax(): + if request.headers.get("x-requested-with") == "XMLHttpRequest": res = ( loader.render_to_string("message.html", context=context, request=request) .replace("'", r"\'") @@ -915,7 +915,7 @@ def edit(request, username, id_string): # noqa C901 xform.update() - if request.is_ajax(): + if request.headers.get("x-requested-with") == "XMLHttpRequest": return HttpResponse(_("Updated succeeded.")) return HttpResponseRedirect( reverse(show, kwargs={"username": username, "id_string": id_string}) @@ -1327,7 +1327,7 @@ def set_perm(request, username, id_string): # noqa C901 request, ) - if request.is_ajax(): + if request.headers.get("x-requested-with") == "XMLHttpRequest": return JsonResponse({"status": "success"}) return HttpResponseRedirect( diff --git a/onadata/apps/restservice/views.py b/onadata/apps/restservice/views.py index e8e1a4c5c3..5af8b213ab 100644 --- a/onadata/apps/restservice/views.py +++ b/onadata/apps/restservice/views.py @@ -63,7 +63,7 @@ def add_service(request, username, id_string): message += Template("{{ field.errors }}").render( Context({"field": field}) ) - if request.is_ajax(): + if request.headers.get("x-requested-with") == "XMLHttpRequest": response = {"status": status, "message": message} if restservice: response["restservice"] = f"{restservice}" diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 7ae8f6b66f..8b8c05a22f 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -971,7 +971,7 @@ def charts(request, username, id_string): summaries = build_chart_data(xform, lang_index, page) - if request.is_ajax(): + if request.headers.get("x-requested-with") == "XMLHttpRequest": template = "charts_snippet.html" else: template = "charts.html" From 4a7b9ce1d154b7a966e57cf6371ac84b9e865a7c Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 17:12:28 +0300 Subject: [PATCH 09/37] Django 4: The {% ifequal %} and {% ifnotequal %} template tags are removed. --- onadata/libs/templates/change_language.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/templates/change_language.html b/onadata/libs/templates/change_language.html index 7840cddf1a..31e27c9d1c 100644 --- a/onadata/libs/templates/change_language.html +++ b/onadata/libs/templates/change_language.html @@ -5,7 +5,7 @@ From 891a28c050dbfd9ade0e1b85442773e29ddcfddd Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Thu, 21 Mar 2024 18:24:27 +0300 Subject: [PATCH 10/37] Remove unused import --- onadata/apps/logger/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onadata/apps/logger/views.py b/onadata/apps/logger/views.py index 1308b8a900..ed76b0d935 100644 --- a/onadata/apps/logger/views.py +++ b/onadata/apps/logger/views.py @@ -4,7 +4,6 @@ """ import os import tempfile -from datetime import datetime from django.conf import settings from django.contrib import messages From 0e5419b844abaa32862dc0b466a56aa495590091 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 12:42:30 +0300 Subject: [PATCH 11/37] Django 4: do not use never_cache decorator with DRF DRF requests are not HttpRequest objects --- onadata/apps/api/viewsets/connect_viewset.py | 7 ------- onadata/apps/api/viewsets/xform_list_viewset.py | 2 -- onadata/apps/api/viewsets/xform_viewset.py | 2 -- 3 files changed, 11 deletions(-) diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index 549a26fe55..75c90fe857 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -8,7 +8,6 @@ from django.utils import timezone from django.utils.decorators import classonlymethod from django.utils.translation import gettext as _ -from django.views.decorators.cache import never_cache from multidb.pinning import use_master from rest_framework import mixins, status, viewsets @@ -203,9 +202,3 @@ def odk_token(self, request, *args, **kwargs): data={"odk_token": token.raw_key, "expires": token.expires}, status=status_code, ) - - @classonlymethod - def as_view(cls, actions=None, **initkwargs): # noqa - view = super(ConnectViewSet, cls).as_view(actions, **initkwargs) - - return never_cache(view) diff --git a/onadata/apps/api/viewsets/xform_list_viewset.py b/onadata/apps/api/viewsets/xform_list_viewset.py index 5144f27f5e..e802cd9897 100644 --- a/onadata/apps/api/viewsets/xform_list_viewset.py +++ b/onadata/apps/api/viewsets/xform_list_viewset.py @@ -5,7 +5,6 @@ from django.conf import settings from django.http import Http404, StreamingHttpResponse from django.shortcuts import get_object_or_404 -from django.views.decorators.cache import never_cache from django_filters import rest_framework as django_filter_filters from rest_framework import permissions, viewsets @@ -151,7 +150,6 @@ def filter_queryset(self, queryset): return queryset - @never_cache def list(self, request, *args, **kwargs): # pylint: disable=attribute-defined-outside-init self.object_list = self.filter_queryset(self.get_queryset()) diff --git a/onadata/apps/api/viewsets/xform_viewset.py b/onadata/apps/api/viewsets/xform_viewset.py index fb79f2ad89..a4b5d5f3e2 100644 --- a/onadata/apps/api/viewsets/xform_viewset.py +++ b/onadata/apps/api/viewsets/xform_viewset.py @@ -26,7 +26,6 @@ from django.utils import timezone from django.utils.http import urlencode from django.utils.translation import gettext as _ -from django.views.decorators.cache import never_cache import six from django_filters.rest_framework import DjangoFilterBackend @@ -406,7 +405,6 @@ def create_async(self, request, *args, **kwargs): return Response(data=resp, status=resp_code, headers=headers) @action(methods=["GET", "HEAD"], detail=True) - @never_cache def form(self, request, **kwargs): """Returns the XLSForm in any of JSON, XML, XLS(X), CSV formats.""" form = self.get_object() From 0a586b5d23e8a18713e68706599fc9b33566d135 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 13:15:00 +0300 Subject: [PATCH 12/37] test submission time should be considered to be in UTC already --- onadata/apps/main/tests/test_process.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 9af3918aa4..113e575403 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -77,9 +77,9 @@ def _update_dynamic_data(self): """ for uuid, submission_time in iteritems(self.uuid_to_submission_times): i = self.xform.instances.get(uuid=uuid) - i.date_created = datetime.strptime( - submission_time, MONGO_STRFTIME - ).astimezone(timezone.utc) + i.date_created = datetime.strptime(submission_time, MONGO_STRFTIME).replace( + tzinfo=timezone.utc + ) i.json = i.get_full_dict() i.save() From 9a195ff1d4b7b47b73cb353bf47c831f247fdd80 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 14:15:46 +0300 Subject: [PATCH 13/37] remove unused import --- onadata/apps/api/viewsets/connect_viewset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onadata/apps/api/viewsets/connect_viewset.py b/onadata/apps/api/viewsets/connect_viewset.py index 75c90fe857..a662f782b0 100644 --- a/onadata/apps/api/viewsets/connect_viewset.py +++ b/onadata/apps/api/viewsets/connect_viewset.py @@ -6,7 +6,6 @@ """ from django.core.exceptions import MultipleObjectsReturned from django.utils import timezone -from django.utils.decorators import classonlymethod from django.utils.translation import gettext as _ from multidb.pinning import use_master From fc19c4ef20eef0e0f66440290f984beba38dcbdf Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 14:16:23 +0300 Subject: [PATCH 14/37] Django 4: ping sphinx to version 6.x --- docs/projects.rst | 42 +++++++++++++++++++++--------------------- requirements/base.pip | 4 ++-- requirements/dev.pip | 4 ++-- setup.cfg | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/projects.rst b/docs/projects.rst index f25f20d941..21cb868636 100644 --- a/docs/projects.rst +++ b/docs/projects.rst @@ -204,7 +204,7 @@ You can share a project with a user or multiple users by ``PUT`` a payload with Example 1: Sharing with a specific user -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: curl -X PUT -d username=alice -d role=readonly https://api.ona.io/api/v1/projects/1/share @@ -215,8 +215,8 @@ Response HTTP 204 NO CONTENT -Example 2: Sharing with mutliple users -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Example 2: Sharing with more than one user +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ :: curl -X PUT -d username=alice,jake -d role=readonly https://api.ona.io/api/v1/projects/1/share @@ -528,14 +528,14 @@ Example ^^^^^^^ :: - + curl -X GET https://api.ona.io/api/v1/projects/1/invitations Response ^^^^^^^^ :: - + [ { "id": 1, @@ -571,16 +571,16 @@ Example ^^^^^^^ :: - + curl -X GET https://api.ona.io/api/v1/projects/1/invitations?status=2 Response ^^^^^^^^ :: - + [ - + { "id": 2, "email":"johndoe@example.com", @@ -604,7 +604,7 @@ Example ^^^^^^^ :: - + curl -X POST -d "email=janedoe@example.com" -d "role=readonly" https://api.ona.io/api/v1/projects/1/invitations @@ -629,16 +629,16 @@ Response ^^^^^^^^ :: - + { "id": 1, "email": "janedoe@example.com", "role": "readonly", "status": 1, } - -The link embedded in the email will be of the format ``http://{url}`` + +The link embedded in the email will be of the format ``http://{url}`` where: - ``url`` - is the URL the recipient will be redirected to on clicking the link. The default is ``{domain}/api/v1/profiles`` where ``domain`` is domain where the API is hosted. @@ -667,14 +667,14 @@ Example ^^^^^^^ :: - + curl -X PUT -d "email=janedoe@example.com" -d "role=editor" -d "invitation_id=1" https://api.ona.io/api/v1/projects/1/invitations/1 Response ^^^^^^^^ :: - + { "id": 1, "email": "janedoe@example.com", @@ -696,11 +696,11 @@ Example ^^^^^^^ :: - + curl -X POST -d "invitation_id=6" https://api.ona.io/api/v1/projects/1/resend-invitation -``invitation_id``: The primary key of the ``ProjectInvitation`` to resend. +``invitation_id``: The primary key of the ``ProjectInvitation`` to resend. - Must be a ``ProjectInvitation`` whose status is **Pending** @@ -708,7 +708,7 @@ Response ^^^^^^^^ :: - + { "message": "Success" } @@ -727,10 +727,10 @@ Example ^^^^^^^ :: - + curl -X POST -d "invitation_id=6" https://api.ona.io/api/v1/projects/1/revoke-invitation -``invitation_id``: The primary key of the ``ProjectInvitation`` to resend. +``invitation_id``: The primary key of the ``ProjectInvitation`` to resend. - Must be a ``ProjectInvitation`` whose status is **Pending** @@ -738,7 +738,7 @@ Response ^^^^^^^^ :: - + { "message": "Success" } @@ -751,4 +751,4 @@ Since a project invitation is sent to an unregistered user, acceptance of the in when `creating a new user `_. All pending invitations whose email match the new user's email will be accepted and projects shared with the -user \ No newline at end of file +user diff --git a/requirements/base.pip b/requirements/base.pip index 0cfd03d93e..7f2fd6a14b 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -175,7 +175,7 @@ djangorestframework-xml==2.0.0 # via onadata dnspython==2.6.1 # via pymongo -docutils==0.20.1 +docutils==0.19 # via sphinx dpath==2.1.6 # via onadata @@ -358,7 +358,7 @@ six==1.16.0 # tableschema snowballstemmer==2.2.0 # via sphinx -sphinx==7.2.6 +sphinx==6.2.1 # via onadata sphinxcontrib-applehelp==1.0.8 # via sphinx diff --git a/requirements/dev.pip b/requirements/dev.pip index e3d8a9708c..1c6e78f3d0 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -196,7 +196,7 @@ djangorestframework-xml==2.0.0 # via onadata dnspython==2.6.1 # via pymongo -docutils==0.20.1 +docutils==0.19 # via sphinx dodgy==0.2.1 # via prospector @@ -504,7 +504,7 @@ snowballstemmer==2.2.0 # via # pydocstyle # sphinx -sphinx==7.2.6 +sphinx==6.2.1 # via onadata sphinxcontrib-applehelp==1.0.8 # via sphinx diff --git a/setup.cfg b/setup.cfg index bc7909c944..cea9d0d529 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,7 +75,7 @@ install_requires = #XML Instance API utility xmltodict #docs - sphinx + sphinx>=6.2,<7 Markdown #others unicodecsv From 3575023889373dd3460e2d7c439c49dec3f97085 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 17:38:01 +0300 Subject: [PATCH 15/37] Ensure date to date comparison for delete cache values --- onadata/apps/logger/models/instance.py | 10 +++++----- onadata/apps/logger/models/xform.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/onadata/apps/logger/models/instance.py b/onadata/apps/logger/models/instance.py index 5ee33bc8be..5b87600ebf 100644 --- a/onadata/apps/logger/models/instance.py +++ b/onadata/apps/logger/models/instance.py @@ -186,17 +186,17 @@ def _update_submission_count_for_today( form_id: int, incr: bool = True, date_created=None ): # Track submissions made today - current_date = timezone.localtime().isoformat() - date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}" f"{form_id}" + current_date = timezone.localdate().isoformat() + date_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{form_id}" count_cache_key = f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{form_id}" if not cache.get(date_cache_key) == current_date: cache.set(date_cache_key, current_date, 86400) if date_created: - date_created = date_created.astimezone( - timezone.get_current_timezone() - ).isoformat() + date_created = ( + date_created.astimezone(timezone.get_current_timezone()).date().isoformat() + ) current_count = cache.get(count_cache_key) if not current_count and incr: diff --git a/onadata/apps/logger/models/xform.py b/onadata/apps/logger/models/xform.py index 195b304de5..546d19c5dd 100644 --- a/onadata/apps/logger/models/xform.py +++ b/onadata/apps/logger/models/xform.py @@ -1158,7 +1158,7 @@ def submission_count(self, force_update=False): @property def submission_count_for_today(self): """Returns the submissions count for the current day.""" - current_date = timezone.localtime().isoformat() + current_date = timezone.localdate().isoformat() count = ( cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY}{self.id}") if cache.get(f"{XFORM_SUBMISSION_COUNT_FOR_DAY_DATE}{self.id}") From 338bef1e7c7a5e51a7f987434bcde962e19004a5 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 20:11:17 +0300 Subject: [PATCH 16/37] Django 4: update not found tests --- .../tests/viewsets/test_dataview_viewset.py | 4 +- .../api/tests/viewsets/test_osm_viewset.py | 159 +++++++++--------- 2 files changed, 83 insertions(+), 80 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index 35d9f20f29..d959bd4d44 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -211,7 +211,9 @@ def test_dataview_with_attachment_field(self): ) self.assertEqual(response.status_code, 404) response_data = json.loads(json.dumps(response.data)) - self.assertEqual(response_data, {"detail": "Not found."}) + self.assertEqual( + response_data, {"detail": "No Attachment matches the given query"} + ) # a user with permissions can view a specific attachment object attachment_list_view = AttachmentViewSet.as_view({"get": "retrieve"}) diff --git a/onadata/apps/api/tests/viewsets/test_osm_viewset.py b/onadata/apps/api/tests/viewsets/test_osm_viewset.py index 026bff9d4e..4ba649601c 100644 --- a/onadata/apps/api/tests/viewsets/test_osm_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_osm_viewset.py @@ -11,108 +11,108 @@ from mock import patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.api.viewsets.osm_viewset import OsmViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.logger.models import Attachment, Instance, OsmData from onadata.apps.viewer.models import Export -from onadata.libs.utils.common_tools import (filename_from_disposition, - get_response_content) +from onadata.libs.utils.common_tools import ( + filename_from_disposition, + get_response_content, +) from onadata.libs.utils.osm import save_osm_data class TestOSMViewSet(TestAbstractViewSet): - def setUp(self): super(self.__class__, self).setUp() self._login_user_and_profile() self.factory = RequestFactory() - self.extra = { - 'HTTP_AUTHORIZATION': 'Token %s' % self.user.auth_token} + self.extra = {"HTTP_AUTHORIZATION": "Token %s" % self.user.auth_token} def _publish_osm_with_submission(self): filenames = [ - 'OSMWay234134797.osm', - 'OSMWay34298972.osm', + "OSMWay234134797.osm", + "OSMWay34298972.osm", ] - self.fixtures_dir = osm_fixtures_dir = os.path.realpath(os.path.join( - os.path.dirname(__file__), '..', 'fixtures', 'osm')) - paths = [ - os.path.join(osm_fixtures_dir, filename) - for filename in filenames] - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') - self.combined_osm_path = os.path.join(osm_fixtures_dir, 'combined.osm') + self.fixtures_dir = osm_fixtures_dir = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "fixtures", "osm") + ) + paths = [os.path.join(osm_fixtures_dir, filename) for filename in filenames] + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") + self.combined_osm_path = os.path.join(osm_fixtures_dir, "combined.osm") self._publish_xls_form_to_project(xlsform_path=xlsform_path) - self.xform.version = '201511091147' + self.xform.version = "201511091147" self.xform.save() # look at the forms.json?instances_with_osm=True - request = self.factory.get('/', {'instances_with_osm': 'True'}, - **self.extra) - view = XFormViewSet.as_view({'get': 'list'}) - response = view(request, format='json') + request = self.factory.get("/", {"instances_with_osm": "True"}, **self.extra) + view = XFormViewSet.as_view({"get": "list"}) + response = view(request, format="json") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, []) - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") files = [open(path) for path in paths] - count = Attachment.objects.filter(extension='osm').count() + count = Attachment.objects.filter(extension="osm").count() count_osm = OsmData.objects.count() - _submission_time = parse_datetime('2013-02-18 15:54:01Z') - self._make_submission(submission_path, media_file=files, - forced_submission_time=_submission_time) - self.assertTrue( - Attachment.objects.filter(extension='osm').count() > count) + _submission_time = parse_datetime("2013-02-18 15:54:01Z") + self._make_submission( + submission_path, media_file=files, forced_submission_time=_submission_time + ) + self.assertTrue(Attachment.objects.filter(extension="osm").count() > count) self.assertEqual(OsmData.objects.count(), count_osm + 2) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_data_retrieve_instance_osm_format(self): self._publish_osm_with_submission() formid = self.xform.pk - dataid = self.xform.instances.latest('date_created').pk - request = self.factory.get('/') + dataid = self.xform.instances.latest("date_created").pk + request = self.factory.get("/") # look at the data/[pk]/[dataid].osm endpoint - view = OsmViewSet.as_view({'get': 'retrieve'}) - response = view(request, pk=formid, dataid=dataid, format='osm') + view = OsmViewSet.as_view({"get": "retrieve"}) + response = view(request, pk=formid, dataid=dataid, format="osm") self.assertEqual(response.status_code, 200) with open(self.combined_osm_path) as f: osm = f.read() response.render() - self.assertMultiLineEqual(response.content.decode('utf-8').strip(), - osm.strip()) + self.assertMultiLineEqual( + response.content.decode("utf-8").strip(), osm.strip() + ) # look at the data/[pk].osm endpoint - view = OsmViewSet.as_view({'get': 'list'}) - response = view(request, pk=formid, format='osm') + view = OsmViewSet.as_view({"get": "list"}) + response = view(request, pk=formid, format="osm") self.assertEqual(response.status_code, 200) response.render() - self.assertMultiLineEqual(response.content.decode('utf-8').strip(), - osm.strip()) + self.assertMultiLineEqual( + response.content.decode("utf-8").strip(), osm.strip() + ) # look at the data.osm endpoint - view = OsmViewSet.as_view({'get': 'list'}) - response = view(request, format='osm') + view = OsmViewSet.as_view({"get": "list"}) + response = view(request, format="osm") self.assertEqual(response.status_code, 301) - self.assertEqual(response['Location'], - 'http://testserver/api/v1/osm.json') + self.assertEqual(response["Location"], "http://testserver/api/v1/osm.json") - response = view(request, format='json') + response = view(request, format="json") self.assertEqual(response.status_code, 200) - data = [{ - 'url': 'http://testserver/api/v1/osm/{}'.format(self.xform.pk), - 'title': self.xform.title, - 'id_string': self.xform.id_string, 'user': self.xform.user.username - }] + data = [ + { + "url": "http://testserver/api/v1/osm/{}".format(self.xform.pk), + "title": self.xform.title, + "id_string": self.xform.id_string, + "user": self.xform.user.username, + } + ] self.assertEqual(response.data, data) # look at the forms.json?instances_with_osm=True - request = self.factory.get('/', {'instances_with_osm': 'True'}, - **self.extra) - view = XFormViewSet.as_view({'get': 'list'}) - response = view(request, format='json') + request = self.factory.get("/", {"instances_with_osm": "True"}, **self.extra) + view = XFormViewSet.as_view({"get": "list"}) + response = view(request, format="json") self.assertEqual(response.status_code, 200) self.assertNotEqual(response.data, []) @@ -121,46 +121,46 @@ def test_osm_csv_export(self): self._publish_osm_with_submission() count = Export.objects.all().count() - view = XFormViewSet.as_view({ - 'get': 'retrieve' - }) + view = XFormViewSet.as_view({"get": "retrieve"}) - request = self.factory.get('/', data={'include_images': False}, - **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", data={"include_images": False}, **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) self.assertEqual(count + 1, Export.objects.all().count()) headers = dict(response.items()) - self.assertEqual(headers['Content-Type'], 'application/csv') - content_disposition = headers['Content-Disposition'] + self.assertEqual(headers["Content-Type"], "application/csv") + content_disposition = headers["Content-Disposition"] filename = filename_from_disposition(content_disposition) basename, ext = os.path.splitext(filename) - self.assertEqual(ext, '.csv') + self.assertEqual(ext, ".csv") content = get_response_content(response) reader = csv.DictReader(StringIO(content)) data = [_ for _ in reader] - test_file_path = os.path.join(self.fixtures_dir, 'osm.csv') - with open(test_file_path, 'r') as test_file: + test_file_path = os.path.join(self.fixtures_dir, "osm.csv") + with open(test_file_path, "r") as test_file: expected_csv_reader = csv.DictReader(test_file) for index, row in enumerate(expected_csv_reader): self.assertDictContainsSubset(row, data[index]) - request = self.factory.get('/', **self.extra) - response = view(request, pk=self.xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = view(request, pk=self.xform.pk, format="csv") self.assertEqual(response.status_code, 200) def test_process_error_osm_format(self): self._publish_xls_form_to_project() self._make_submissions() - request = self.factory.get('/') - view = DataViewSet.as_view({'get': 'retrieve'}) - dataid = self.xform.instances.all().order_by('id')[0].pk - response = view(request, pk=self.xform.pk, dataid=dataid, format='osm') - self.assertContains(response, 'Not found.', - status_code=404) + request = self.factory.get("/") + view = DataViewSet.as_view({"get": "retrieve"}) + dataid = self.xform.instances.all().order_by("id")[0].pk + response = view(request, pk=self.xform.pk, dataid=dataid, format="osm") + self.assertContains( + response, + "No data matches with given query.", + status_code=404, + ) def test_save_osm_data_transaction_atomic(self): """ @@ -178,7 +178,7 @@ def test_save_osm_data_transaction_atomic(self): # mock the save method on OsmData and cause it to raise an # IntegrityError on its first call only, so that we get into the # catch inside save_osm_data - with patch('onadata.libs.utils.osm.OsmData.save') as mock: + with patch("onadata.libs.utils.osm.OsmData.save") as mock: def _side_effect(*args): """ @@ -212,23 +212,24 @@ def test_save_osm_data_with_non_existing_media_file(self): Test that saving osm data with a non existing media file fails silenty and does not throw an IOError """ - osm_fixtures_dir = os.path.realpath(os.path.join( - os.path.dirname(__file__), '..', 'fixtures', 'osm')) + osm_fixtures_dir = os.path.realpath( + os.path.join(os.path.dirname(__file__), "..", "fixtures", "osm") + ) # publish form - xlsform_path = os.path.join(osm_fixtures_dir, 'osm.xlsx') + xlsform_path = os.path.join(osm_fixtures_dir, "osm.xlsx") self._publish_xls_form_to_project(xlsform_path=xlsform_path) self.xform.save() # make submission with osm data - submission_path = os.path.join(osm_fixtures_dir, 'instance_a.xml') - media_file = open(os.path.join(osm_fixtures_dir, - 'OSMWay234134797.osm')) + submission_path = os.path.join(osm_fixtures_dir, "instance_a.xml") + media_file = open(os.path.join(osm_fixtures_dir, "OSMWay234134797.osm")) self._make_submission(submission_path, media_file=media_file) # save osm data with a non existing file submission = Instance.objects.first() attachment = submission.attachments.first() attachment.media_file = os.path.join( - settings.PROJECT_ROOT, "test_media", "noFile.osm") + settings.PROJECT_ROOT, "test_media", "noFile.osm" + ) attachment.save() try: save_osm_data(submission.id) From f97d53a0effaf6828d20b270afa7cf238bbec8eb Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 22 Mar 2024 21:53:49 +0300 Subject: [PATCH 17/37] Django 4: +djangorestframework==3.15.1 --- .../api/tests/viewsets/test_user_viewset.py | 120 ++++++++++-------- requirements/base.pip | 10 +- requirements/dev.pip | 10 +- 3 files changed, 77 insertions(+), 63 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_user_viewset.py b/onadata/apps/api/tests/viewsets/test_user_viewset.py index 57461f02ee..63e3da68e1 100644 --- a/onadata/apps/api/tests/viewsets/test_user_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_viewset.py @@ -1,73 +1,81 @@ -from onadata.apps.api.tests.viewsets.test_abstract_viewset import\ - TestAbstractViewSet +# -*- coding: utf-8 -*- +""" +Test UserViewSet module. +""" + +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.user_viewset import UserViewSet class TestUserViewSet(TestAbstractViewSet): - def setUp(self): - super(self.__class__, self).setUp() - self.data = {'id': self.user.pk, 'username': u'bob', - 'first_name': u'Bob', 'last_name': u'erama'} + super().setUp() + self.data = { + "id": self.user.pk, + "username": "bob", + "first_name": "Bob", + "last_name": "erama", + } def test_user_get(self): """Test authenticated user can access user info""" - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) # users list - view = UserViewSet.as_view({'get': 'list'}) + view = UserViewSet.as_view({"get": "list"}) response = view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertIsNone(response.get("Cache-Control")) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [self.data]) # user with username bob - view = UserViewSet.as_view({'get': 'retrieve'}) - response = view(request, username='bob') - self.assertNotEqual(response.get('Cache-Control'), None) + view = UserViewSet.as_view({"get": "retrieve"}) + response = view(request, username="bob") + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) # user with username BoB, mixed case - view = UserViewSet.as_view({'get': 'retrieve'}) - response = view(request, username='BoB') + view = UserViewSet.as_view({"get": "retrieve"}) + response = view(request, username="BoB") self.assertEqual(response.status_code, 200) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.data, self.data) # Test can retrieve profile for usernames with _ . @ symbols alice_data = { - 'username': 'alice.test@gmail.com', 'email': 'alice@localhost.com'} + "username": "alice.test@gmail.com", + "email": "alice@localhost.com", + } alice_profile = self._create_user_profile(alice_data) - extra = { - 'HTTP_AUTHORIZATION': f'Token {alice_profile.user.auth_token}'} + extra = {"HTTP_AUTHORIZATION": f"Token {alice_profile.user.auth_token}"} - request = self.factory.get('/', **extra) - response = view(request, username='alice.test@gmail.com') + request = self.factory.get("/", **extra) + response = view(request, username="alice.test@gmail.com") self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['username'], alice_data['username']) + self.assertEqual(response.data["username"], alice_data["username"]) # user bob is_active = False self.user.is_active = False self.user.save() - view = UserViewSet.as_view({'get': 'retrieve'}) - response = view(request, username='BoB') + view = UserViewSet.as_view({"get": "retrieve"}) + response = view(request, username="BoB") self.assertEqual(response.status_code, 404) def test_user_anon(self): """Test anonymous user can access user info""" - request = self.factory.get('/') + request = self.factory.get("/") # users list endpoint - view = UserViewSet.as_view({'get': 'list'}) + view = UserViewSet.as_view({"get": "list"}) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [self.data]) # user with username bob - view = UserViewSet.as_view({'get': 'retrieve'}) - response = view(request, username='bob') + view = UserViewSet.as_view({"get": "retrieve"}) + response = view(request, username="bob") self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.data) @@ -77,36 +85,44 @@ def test_user_anon(self): self.assertEqual(response.data, self.data) def test_get_user_using_email(self): - alice_data = {'username': 'alice', 'email': 'alice@localhost.com', - 'first_name': u'Alice', 'last_name': u'Kamande'} + alice_data = { + "username": "alice", + "email": "alice@localhost.com", + "first_name": "Alice", + "last_name": "Kamande", + } alice_profile = self._create_user_profile(alice_data) - data = [{'id': alice_profile.user.pk, 'username': u'alice', - 'first_name': u'Alice', 'last_name': u'Kamande'}] + data = [ + { + "id": alice_profile.user.pk, + "username": "alice", + "first_name": "Alice", + "last_name": "Kamande", + } + ] get_params = { - 'search': alice_profile.user.email, + "search": alice_profile.user.email, } - view = UserViewSet.as_view( - {'get': 'list'} - ) - request = self.factory.get('/', data=get_params) + view = UserViewSet.as_view({"get": "list"}) + request = self.factory.get("/", data=get_params) response = view(request) self.assertEqual(response.status_code, 401) - error = {'detail': 'Authentication credentials were not provided.'} + error = {"detail": "Authentication credentials were not provided."} self.assertEqual(response.data, error) # authenticated - request = self.factory.get('/', data=get_params, **self.extra) + request = self.factory.get("/", data=get_params, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, data) get_params = { - 'search': 'doesnotexist@email.com', + "search": "doesnotexist@email.com", } - request = self.factory.get('/', data=get_params, **self.extra) + request = self.factory.get("/", data=get_params, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -114,10 +130,10 @@ def test_get_user_using_email(self): self.assertEqual(response.data, []) get_params = { - 'search': 'invalid@email.com', + "search": "invalid@email.com", } - request = self.factory.get('/', data=get_params, **self.extra) + request = self.factory.get("/", data=get_params, **self.extra) response = view(request) self.assertEqual(response.status_code, 200) @@ -127,22 +143,20 @@ def test_get_user_using_email(self): def test_get_non_org_users(self): self._org_create() - view = UserViewSet.as_view( - {'get': 'list'} - ) + view = UserViewSet.as_view({"get": "list"}) - all_users_request = self.factory.get('/') + all_users_request = self.factory.get("/") all_users_response = view(all_users_request) self.assertEqual(all_users_response.status_code, 200) - self.assertEqual(len( - [u for u in all_users_response.data if u['username'] == 'denoinc'] - ), 1) + self.assertEqual( + len([u for u in all_users_response.data if u["username"] == "denoinc"]), 1 + ) - no_orgs_request = self.factory.get('/', data={'orgs': 'false'}) + no_orgs_request = self.factory.get("/", data={"orgs": "false"}) no_orgs_response = view(no_orgs_request) self.assertEqual(no_orgs_response.status_code, 200) - self.assertEqual(len( - [u for u in no_orgs_response.data if u['username'] == 'denoinc']), - 0) + self.assertEqual( + len([u for u in no_orgs_response.data if u["username"] == "denoinc"]), 0 + ) diff --git a/requirements/base.pip b/requirements/base.pip index 7f2fd6a14b..81b9beb2cc 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -26,7 +26,7 @@ analytics-python==1.4.post1 # via onadata appoptics-metrics==5.1.0 # via onadata -asgiref==3.8.0 +asgiref==3.8.1 # via # django # django-cors-headers @@ -41,9 +41,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.67 +boto3==1.34.68 # via dataflows-tabulator -botocore==1.34.67 +botocore==1.34.68 # via # boto3 # s3transfer @@ -153,7 +153,7 @@ django-taggit==4.0.0 # via onadata django-templated-email==3.0.1 # via onadata -djangorestframework==3.15.0 +djangorestframework==3.15.1 # via # djangorestframework-csv # djangorestframework-gis @@ -378,7 +378,7 @@ sqlparse==0.4.4 # via # django # django-debug-toolbar -tableschema==1.20.9 +tableschema==1.20.10 # via datapackage typing-extensions==4.10.0 # via diff --git a/requirements/dev.pip b/requirements/dev.pip index 1c6e78f3d0..89902aa040 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -26,7 +26,7 @@ analytics-python==1.4.post1 # via onadata appoptics-metrics==5.1.0 # via onadata -asgiref==3.8.0 +asgiref==3.8.1 # via # django # django-cors-headers @@ -49,9 +49,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.67 +boto3==1.34.68 # via dataflows-tabulator -botocore==1.34.67 +botocore==1.34.68 # via # boto3 # s3transfer @@ -174,7 +174,7 @@ django-taggit==4.0.0 # via onadata django-templated-email==3.0.1 # via onadata -djangorestframework==3.15.0 +djangorestframework==3.15.1 # via # djangorestframework-csv # djangorestframework-gis @@ -526,7 +526,7 @@ sqlparse==0.4.4 # django-debug-toolbar stack-data==0.6.3 # via ipython -tableschema==1.20.9 +tableschema==1.20.10 # via datapackage tblib==3.0.0 # via -r requirements/dev.in From e7c8b413e1bb96ad7ee9f661cfe46b75848fd1d3 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 10:27:00 +0300 Subject: [PATCH 18/37] Add missing dot in test error message --- onadata/apps/api/tests/viewsets/test_dataview_viewset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index d959bd4d44..6b28323406 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -212,7 +212,7 @@ def test_dataview_with_attachment_field(self): self.assertEqual(response.status_code, 404) response_data = json.loads(json.dumps(response.data)) self.assertEqual( - response_data, {"detail": "No Attachment matches the given query"} + response_data, {"detail": "No Attachment matches the given query."} ) # a user with permissions can view a specific attachment object From 770eccbb848982f1992c75b24b63b6d4b1a9d92b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 10:28:34 +0300 Subject: [PATCH 19/37] User endpoint has cache headers due to removal of never_cache decorator --- onadata/apps/api/tests/viewsets/test_user_viewset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_user_viewset.py b/onadata/apps/api/tests/viewsets/test_user_viewset.py index 63e3da68e1..7a76b45cec 100644 --- a/onadata/apps/api/tests/viewsets/test_user_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_viewset.py @@ -24,7 +24,6 @@ def test_user_get(self): # users list view = UserViewSet.as_view({"get": "list"}) response = view(request) - self.assertIsNone(response.get("Cache-Control")) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [self.data]) From 24ad97009460500e1beaebaeeee66b084ff44680 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 12:11:28 +0300 Subject: [PATCH 20/37] Switch to the standard python library unittest.mock --- .../api/tests/permissions/test_permissions.py | 31 +- .../tests/viewsets/test_connect_viewset.py | 12 +- .../api/tests/viewsets/test_data_viewset.py | 8 +- .../tests/viewsets/test_dataview_viewset.py | 27 +- .../api/tests/viewsets/test_media_viewset.py | 8 +- .../tests/viewsets/test_metadata_viewset.py | 548 +++++++++--------- .../test_organization_profile_viewset.py | 2 +- .../api/tests/viewsets/test_osm_viewset.py | 11 +- .../tests/viewsets/test_project_viewset.py | 2 +- .../test_submission_review_viewset.py | 15 +- .../viewsets/test_user_profile_viewset.py | 9 +- .../tests/viewsets/test_xform_list_viewset.py | 13 +- .../api/tests/viewsets/test_xform_viewset.py | 4 +- .../apps/logger/tests/models/test_instance.py | 2 +- .../apps/logger/tests/test_form_submission.py | 18 +- onadata/apps/main/tests/test_process.py | 2 +- onadata/apps/main/tests/test_user_profile.py | 142 +++-- .../messaging/tests/test_backends_mqtt.py | 158 ++--- onadata/apps/messaging/tests/test_signals.py | 34 +- .../restservice/tests/test_restservice.py | 3 +- .../viewsets/test_restservicesviewset.py | 4 +- .../apps/viewer/tests/test_attachment_url.py | 50 +- onadata/apps/viewer/tests/test_exports.py | 24 +- onadata/libs/tests/data/test_statistics.py | 8 + onadata/libs/tests/data/test_tools.py | 16 +- .../serializers/test_project_serializer.py | 156 ++--- .../serializers/test_xform_serializer.py | 11 +- onadata/libs/tests/test_authentication.py | 11 +- onadata/libs/tests/test_permissions.py | 119 ++-- onadata/libs/tests/utils/test_csv_import.py | 29 +- onadata/libs/tests/utils/test_email.py | 20 +- .../libs/tests/utils/test_export_builder.py | 2 +- onadata/libs/tests/utils/test_logger_tools.py | 2 +- .../libs/tests/utils/test_project_utils.py | 3 +- onadata/libs/tests/utils/test_viewer_tools.py | 3 +- requirements/base.pip | 12 +- requirements/dev.in | 1 - requirements/dev.pip | 14 +- setup.cfg | 1 - 39 files changed, 806 insertions(+), 729 deletions(-) diff --git a/onadata/apps/api/tests/permissions/test_permissions.py b/onadata/apps/api/tests/permissions/test_permissions.py index c2e499e69c..040af9aedb 100644 --- a/onadata/apps/api/tests/permissions/test_permissions.py +++ b/onadata/apps/api/tests/permissions/test_permissions.py @@ -1,14 +1,19 @@ +# -*- coding: utf-8 -*- +""" +Test onadata.apps.api.permissions module. +""" +from unittest.mock import MagicMock, patch + from django.contrib.auth.models import User from django.http import Http404 -from mock import MagicMock, patch -from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet -from onadata.apps.logger.models import Instance, XForm from onadata.apps.api.permissions import ( + AlternateHasObjectPermissionMixin, IsAuthenticatedSubmission, MetaDataObjectPermissions, - AlternateHasObjectPermissionMixin ) +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet +from onadata.apps.logger.models import Instance, XForm from onadata.libs.permissions import UserProfile @@ -25,13 +30,11 @@ def setUp(self): self.instance.xform = MagicMock(XForm) def test_delete_instance_metadata_perms(self): - request = MagicMock(user=MagicMock(), method='DELETE') + request = MagicMock(user=MagicMock(), method="DELETE") obj = MagicMock(content_object=self.instance) - self.assertTrue( - self.permissions.has_object_permission( - request, self.view, obj)) + self.assertTrue(self.permissions.has_object_permission(request, self.view, obj)) - @patch.object(AlternateHasObjectPermissionMixin, '_has_object_permission') + @patch.object(AlternateHasObjectPermissionMixin, "_has_object_permission") def test_delete_instance_metadata_without_perms(self, has_perms_mock): """ Test that a user cannot delete an instance if they are not allowed @@ -41,11 +44,11 @@ def test_delete_instance_metadata_without_perms(self, has_perms_mock): user = User(username="test") instance = Instance(user=User(username="username")) instance.xform = XForm() - request = MagicMock(user=user, method='DELETE') + request = MagicMock(user=user, method="DELETE") obj = MagicMock(content_object=instance) self.assertFalse( - self.permissions.has_object_permission( - request, self.view, obj)) + self.permissions.has_object_permission(request, self.view, obj) + ) def test_is_authenticated_submission_permissions(self): """ @@ -56,11 +59,11 @@ def test_is_authenticated_submission_permissions(self): project = self.xform.project submission_permission = IsAuthenticatedSubmission() - request = MagicMock(method='GET') + request = MagicMock(method="GET") view = MagicMock(username=user.username) self.assertTrue(submission_permission.has_permission(request, self.view)) - request = MagicMock(method='POST') + request = MagicMock(method="POST") view = MagicMock(kwargs={"username": user.username}) self.assertTrue(submission_permission.has_permission(request, view)) diff --git a/onadata/apps/api/tests/viewsets/test_connect_viewset.py b/onadata/apps/api/tests/viewsets/test_connect_viewset.py index b2b309b755..d6aa51706b 100644 --- a/onadata/apps/api/tests/viewsets/test_connect_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_connect_viewset.py @@ -2,8 +2,8 @@ """ Test /user API endpoint """ -from datetime import datetime -from datetime import timedelta +from datetime import datetime, timedelta +from unittest.mock import patch from django.conf import settings from django.contrib.auth.models import User @@ -12,19 +12,19 @@ from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_encode from django.utils.timezone import now + from django_digest.backend.db import update_partial_digests -from django_digest.test import DigestAuth, BasicAuth -from mock import patch +from django_digest.test import BasicAuth, DigestAuth from rest_framework import authentication from rest_framework.authtoken.models import Token -from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.models.odk_token import ODKToken +from onadata.apps.api.models.temp_token import TempToken from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet -from onadata.libs.serializers.password_reset_serializer import default_token_generator from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.libs.authentication import DigestAuthentication +from onadata.libs.serializers.password_reset_serializer import default_token_generator from onadata.libs.serializers.project_serializer import ProjectSerializer from onadata.libs.utils.cache_tools import safe_key diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index f33dc18ee1..7adadc8b31 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -13,6 +13,7 @@ from datetime import timedelta from io import StringIO from tempfile import NamedTemporaryFile +from unittest.mock import Mock, patch from django.conf import settings from django.core.cache import cache @@ -29,7 +30,6 @@ from django_digest.test import DigestAuth from flaky import flaky from httmock import HTTMock, urlmatch -from mock import Mock, patch from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( TestAbstractViewSet, @@ -1805,7 +1805,7 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): "%d records were deleted" % len(records_to_be_deleted), ) self.assertTrue(send_message_mock.called) - send_message_mock.called_with( + send_message_mock.assert_called_with( [str(i.pk) for i in records_to_be_deleted], formid, XFORM, @@ -1905,7 +1905,7 @@ def test_permanent_deletions_bulk_submissions(self, send_message_mock): "%d records were deleted" % len(records_to_be_deleted), ) self.assertTrue(send_message_mock.called) - send_message_mock.called_with( + send_message_mock.assert_called_with( [str(i.pk) for i in records_to_be_deleted], formid, XFORM, @@ -2058,7 +2058,7 @@ def test_delete_submissions(self, send_message_mock): "%d records were deleted" % len(deleted_instances_subset), ) self.assertTrue(send_message_mock.called) - send_message_mock.called_with( + send_message_mock.assert_called_with( [str(i.pk) for i in deleted_instances_subset], formid, XFORM, diff --git a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py index 6b28323406..3069b655fd 100644 --- a/onadata/apps/api/tests/viewsets/test_dataview_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_dataview_viewset.py @@ -1,33 +1,36 @@ # -*- coding: utf-8 -*- """Test DataViewViewSet""" +import csv import json import os -import csv - from datetime import datetime, timedelta +from unittest.mock import patch + from django.conf import settings -from django.test.utils import override_settings from django.core.cache import cache from django.core.files.storage import default_storage +from django.test.utils import override_settings from django.utils.timezone import utc -from mock import patch + from openpyxl import load_workbook -from onadata.libs.permissions import ReadOnlyRole -from onadata.apps.logger.models.data_view import DataView -from onadata.apps.logger.models import Instance, Attachment -from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet -from onadata.apps.viewer.models.export import Export -from onadata.apps.api.viewsets.project_viewset import ProjectViewSet +from onadata.apps.api.viewsets.attachment_viewset import AttachmentViewSet from onadata.apps.api.viewsets.dataview_viewset import ( DataViewViewSet, + apply_filters, filter_to_field_lookup, get_field_lookup, get_filter_kwargs, - apply_filters, ) from onadata.apps.api.viewsets.note_viewset import NoteViewSet +from onadata.apps.api.viewsets.project_viewset import ProjectViewSet +from onadata.apps.api.viewsets.xform_viewset import XFormViewSet +from onadata.apps.logger.models import Attachment, Instance +from onadata.apps.logger.models.data_view import DataView +from onadata.apps.viewer.models.export import Export +from onadata.libs.permissions import ReadOnlyRole +from onadata.libs.serializers.attachment_serializer import AttachmentSerializer from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils.cache_tools import ( DATAVIEW_COUNT, @@ -35,12 +38,10 @@ PROJECT_LINKED_DATAVIEWS, ) from onadata.libs.utils.common_tags import EDITED, MONGO_STRFTIME -from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.libs.utils.common_tools import ( filename_from_disposition, get_response_content, ) -from onadata.libs.serializers.attachment_serializer import AttachmentSerializer class TestDataViewViewSet(TestAbstractViewSet): diff --git a/onadata/apps/api/tests/viewsets/test_media_viewset.py b/onadata/apps/api/tests/viewsets/test_media_viewset.py index 31a674c155..37f3623e38 100644 --- a/onadata/apps/api/tests/viewsets/test_media_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_media_viewset.py @@ -1,8 +1,13 @@ +# -*- coding: utf-8 -*- +""" +Tests the MediaViewSet. +""" +# pylint: disable=too-many-lines import os import urllib +from unittest.mock import MagicMock, patch from django.utils import timezone -from mock import MagicMock, patch from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.media_viewset import MediaViewSet @@ -103,7 +108,6 @@ def test_returned_media_is_based_on_form_perms(self): @patch("onadata.libs.utils.image_tools.get_storage_class") @patch("onadata.libs.utils.image_tools.boto3.client") def test_retrieve_view_from_s3(self, mock_presigned_urls, mock_get_storage_class): - expected_url = ( "https://testing.s3.amazonaws.com/doe/attachments/" "4_Media_file/media.png?" diff --git a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py index a87ff665f1..d81c8b5c5a 100644 --- a/onadata/apps/api/tests/viewsets/test_metadata_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_metadata_viewset.py @@ -1,38 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Tests the MetaDataViewSet. +""" +# pylint: disable=too-many-lines import os from builtins import open -from mock import patch +from unittest.mock import patch from django.conf import settings from django.contrib.contenttypes.models import ContentType from django.core.files.uploadedfile import InMemoryUploadedFile -from onadata.apps.api.tests.viewsets.test_abstract_viewset import ( - TestAbstractViewSet) +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.metadata_viewset import MetaDataViewSet from onadata.apps.api.viewsets.project_viewset import ProjectViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet from onadata.apps.main.models.meta_data import MetaData -from onadata.libs.permissions import (DataEntryRole, DataEntryOnlyRole, - EditorRole, EditorMinorRole) +from onadata.libs.permissions import ( + DataEntryOnlyRole, + DataEntryRole, + EditorMinorRole, + EditorRole, +) from onadata.libs.serializers.metadata_serializer import UNIQUE_TOGETHER_ERROR from onadata.libs.serializers.xform_serializer import XFormSerializer from onadata.libs.utils.common_tags import XFORM_META_PERMS class TestMetaDataViewSet(TestAbstractViewSet): + """ + Tests the MetaDataViewSet. + """ def setUp(self): super(TestMetaDataViewSet, self).setUp() - self.view = MetaDataViewSet.as_view({ - 'delete': 'destroy', - 'get': 'retrieve', - 'post': 'create' - }) + self.view = MetaDataViewSet.as_view( + {"delete": "destroy", "get": "retrieve", "post": "create"} + ) self._publish_xls_form_to_project() self.data_value = "screenshot.png" self.fixture_dir = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation" + settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "transportation" ) self.path = os.path.join(self.fixture_dir, self.data_value) @@ -40,189 +48,191 @@ def setUp(self): ContentType.objects.get_or_create(app_label="logger", model="instance") def _add_project_metadata(self, project, data_type, data_value, path=None): - data = { - 'data_type': data_type, - 'data_value': data_value, - 'project': project.id - } + data = {"data_type": data_type, "data_value": data_value, "project": project.id} if path and data_value: - with open(path, 'rb') as media_file: - data.update({ - 'data_file': media_file, - }) + with open(path, "rb") as media_file: + data.update( + { + "data_file": media_file, + } + ) return self._post_metadata(data) else: return self._post_metadata(data) - def _add_instance_metadata(self, - data_type, - data_value, - path=None): + def _add_instance_metadata(self, data_type, data_value, path=None): xls_file_path = os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "tutorial", "tutorial.xlsx") + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "tutorial", + "tutorial.xlsx", + ) self._publish_xls_form_to_project(xlsform_path=xls_file_path) xml_submission_file_path = os.path.join( - settings.PROJECT_ROOT, "apps", "logger", "fixtures", - "tutorial", "instances", "tutorial_2012-06-27_11-27-53.xml") + settings.PROJECT_ROOT, + "apps", + "logger", + "fixtures", + "tutorial", + "instances", + "tutorial_2012-06-27_11-27-53.xml", + ) - self._make_submission(xml_submission_file_path, - username=self.user.username) + self._make_submission(xml_submission_file_path, username=self.user.username) self.xform.refresh_from_db() self.instance = self.xform.instances.first() data = { - 'data_type': data_type, - 'data_value': data_value, - 'instance': self.instance.id + "data_type": data_type, + "data_value": data_value, + "instance": self.instance.id, } if path and data_value: - with open(path, 'rb') as media_file: - data.update({ - 'data_file': media_file, - }) + with open(path, "rb") as media_file: + data.update( + { + "data_file": media_file, + } + ) self._post_metadata(data) else: self._post_metadata(data) def test_add_metadata_with_file_attachment(self): - for data_type in ['supporting_doc', 'media', 'source']: - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) + for data_type in ["supporting_doc", "media", "source"]: + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) def test_parse_error_is_raised(self): """Parse error is raised when duplicate media is uploaded""" data_type = "supporting_doc" - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) # Duplicate upload - response = self._add_form_metadata(self.xform, data_type, - self.data_value, self.path, False) + response = self._add_form_metadata( + self.xform, data_type, self.data_value, self.path, False + ) self.assertEqual(response.status_code, 400) self.assertIn(UNIQUE_TOGETHER_ERROR, response.data) def test_forms_endpoint_with_metadata(self): date_modified = self.xform.date_modified - for data_type in ['supporting_doc', 'media', 'source']: - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) + for data_type in ["supporting_doc", "media", "source"]: + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) self.xform.refresh_from_db() self.assertNotEqual(date_modified, self.xform.date_modified) # /forms - view = XFormViewSet.as_view({ - 'get': 'retrieve' - }) + view = XFormViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = XFormSerializer(self.xform, context={'request': request}).data + data = XFormSerializer(self.xform, context={"request": request}).data self.assertEqual(response.data, data) # /projects/[pk]/forms - view = ProjectViewSet.as_view({ - 'get': 'forms' - }) - request = self.factory.get('/', **self.extra) + view = ProjectViewSet.as_view({"get": "forms"}) + request = self.factory.get("/", **self.extra) response = view(request, pk=self.project.pk) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, [data]) - @patch('onadata.libs.serializers.metadata_serializer.is_azure_storage') - @patch('azure.storage.blob.generate_blob_sas') + @patch("onadata.libs.serializers.metadata_serializer.is_azure_storage") + @patch("azure.storage.blob.generate_blob_sas") def test_forms_endpoint_with_metadata_and_azure_storage( - self, mock_generate_blob_sas, mock_is_azure_storage): - sas_token = 'sc=date+randomText' + self, mock_generate_blob_sas, mock_is_azure_storage + ): + sas_token = "sc=date+randomText" mock_is_azure_storage.return_value = True mock_generate_blob_sas.return_value = sas_token - self._add_form_metadata(self.xform, 'media', - self.data_value, self.path) + self._add_form_metadata(self.xform, "media", self.data_value, self.path) self.xform.refresh_from_db() # /forms - view = XFormViewSet.as_view({ - 'get': 'retrieve' - }) + view = XFormViewSet.as_view({"get": "retrieve"}) formid = self.xform.pk - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = view(request, pk=formid) self.assertEqual(response.status_code, 200) - data = XFormSerializer(self.xform, context={'request': request}).data + data = XFormSerializer(self.xform, context={"request": request}).data self.assertEqual(response.data, data) self.assertIn(f"?{sas_token}", str(data)) def test_get_metadata_with_file_attachment(self): - for data_type in ['supporting_doc', 'media', 'source']: - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) - request = self.factory.get('/', **self.extra) + for data_type in ["supporting_doc", "media", "source"]: + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.metadata.pk) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) self.assertEqual(response.data, self.metadata_data) - ext = self.data_value[self.data_value.rindex('.') + 1:] - request = self.factory.get('/', **self.extra) + ext = self.data_value[self.data_value.rindex(".") + 1 :] + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.metadata.pk, format=ext) self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Type'], 'image/png') + self.assertEqual(response["Content-Type"], "image/png") def test_get_metadata(self): self.fixture_dir = os.path.join( - settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", - "transportation", "instances", "transport_2011-07-25_19-05-49" + settings.PROJECT_ROOT, + "apps", + "main", + "tests", + "fixtures", + "transportation", + "instances", + "transport_2011-07-25_19-05-49", ) - self.data_value = '1335783522563.jpg' + self.data_value = "1335783522563.jpg" self.path = os.path.join(self.fixture_dir, self.data_value) - self._add_form_metadata( - self.xform, "media", self.data_value, self.path) + self._add_form_metadata(self.xform, "media", self.data_value, self.path) data = { - 'id': self.metadata.pk, - 'xform': self.xform.pk, - 'data_value': u'1335783522563.jpg', - 'data_type': u'media', - 'extra_data': None, - 'data_file': u'http://localhost:8000/media/%s/formid-media/' - '1335783522563.jpg' % self.user.username, - 'data_file_type': u'image/jpeg', - 'media_url': u'http://localhost:8000/media/%s/formid-media/' - '1335783522563.jpg' % self.user.username, - 'file_hash': u'md5:2ca0d22073a9b6b4ebe51368b08da60c', - 'url': 'http://testserver/api/v1/metadata/%s' % self.metadata.pk, - 'date_created': self.metadata.date_created + "id": self.metadata.pk, + "xform": self.xform.pk, + "data_value": "1335783522563.jpg", + "data_type": "media", + "extra_data": None, + "data_file": "http://localhost:8000/media/%s/formid-media/" + "1335783522563.jpg" % self.user.username, + "data_file_type": "image/jpeg", + "media_url": "http://localhost:8000/media/%s/formid-media/" + "1335783522563.jpg" % self.user.username, + "file_hash": "md5:2ca0d22073a9b6b4ebe51368b08da60c", + "url": "http://testserver/api/v1/metadata/%s" % self.metadata.pk, + "date_created": self.metadata.date_created, } - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request, pk=self.metadata.pk) self.assertEqual(response.status_code, 200) self.assertDictEqual(dict(response.data), data) def test_add_mapbox_layer(self): - data_type = 'mapbox_layer' - data_value = 'test_mapbox_layer||http://0.0.0.0:8080||attribution' + data_type = "mapbox_layer" + data_value = "test_mapbox_layer||http://0.0.0.0:8080||attribution" self._add_form_metadata(self.xform, data_type, data_value) def test_delete_metadata(self): - for data_type in ['supporting_doc', 'media', 'source']: + for data_type in ["supporting_doc", "media", "source"]: count = MetaData.objects.count() - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) - request = self.factory.delete('/', **self.extra) + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) + request = self.factory.delete("/", **self.extra) response = self.view(request, pk=self.metadata.pk) self.assertEqual(response.status_code, 204) self.assertEqual(count, MetaData.objects.count()) def test_delete_xform_deletes_media_metadata(self): self._add_test_metadata() - self.view = MetaDataViewSet.as_view({'get': 'list'}) - data = {'xform': self.xform.pk} - request = self.factory.get('/', data, **self.extra) + self.view = MetaDataViewSet.as_view({"get": "list"}) + data = {"xform": self.xform.pk} + request = self.factory.get("/", data, **self.extra) response = self.view(request) meta_count = self.xform.metadata_set.all().count() self.assertEqual(response.status_code, 200) @@ -237,189 +247,188 @@ def test_delete_xform_deletes_media_metadata(self): self.assertEqual(response2.data, []) def test_windows_csv_file_upload_to_metadata(self): - data_value = 'transportation.csv' + data_value = "transportation.csv" path = os.path.join(self.fixture_dir, data_value) with open(path) as f: f = InMemoryUploadedFile( - f, 'media', data_value, 'application/octet-stream', 2625, None) + f, "media", data_value, "application/octet-stream", 2625, None + ) data = { - 'data_value': data_value, - 'data_file': f, - 'data_type': 'media', - 'xform': self.xform.pk + "data_value": data_value, + "data_file": f, + "data_type": "media", + "xform": self.xform.pk, } self._post_metadata(data) - self.assertEqual(self.metadata.data_file_type, 'text/csv') + self.assertEqual(self.metadata.data_file_type, "text/csv") def test_add_media_url(self): - data_type = 'media' + data_type = "media" # test invalid URL - data_value = 'some thing random here' + data_value = "some thing random here" response = self._add_form_metadata( - self.xform, data_type, data_value, test=False) - expected_exception = { - 'data_value': [u"Invalid url 'some thing random here'."] - } + self.xform, data_type, data_value, test=False + ) + expected_exception = {"data_value": ["Invalid url 'some thing random here'."]} self.assertEqual(response.data, expected_exception) # test valid URL - data_value = 'https://devtrac.ona.io/fieldtrips.csv' + data_value = "https://devtrac.ona.io/fieldtrips.csv" self._add_form_metadata(self.xform, data_type, data_value) - request = self.factory.get('/', **self.extra) - ext = self.data_value[self.data_value.rindex('.') + 1:] + request = self.factory.get("/", **self.extra) + ext = self.data_value[self.data_value.rindex(".") + 1 :] response = self.view(request, pk=self.metadata.pk, format=ext) self.assertEqual(response.status_code, 302) - self.assertEqual(response['Location'], data_value) + self.assertEqual(response["Location"], data_value) def test_add_media_xform_link(self): - data_type = 'media' + data_type = "media" # test missing parameters - data_value = 'xform {}'.format(self.xform.pk) + data_value = "xform {}".format(self.xform.pk) response = self._add_form_metadata( - self.xform, data_type, data_value, test=False) + self.xform, data_type, data_value, test=False + ) expected_exception = { - 'data_value': [ - u"Expecting 'xform [xform id] [media name]' or " - "'dataview [dataview id] [media name]' or a valid URL."] + "data_value": [ + "Expecting 'xform [xform id] [media name]' or " + "'dataview [dataview id] [media name]' or a valid URL." + ] } self.assertEqual(response.data, expected_exception) - data_value = 'xform {} transportation'.format(self.xform.pk) + data_value = "xform {} transportation".format(self.xform.pk) self._add_form_metadata(self.xform, data_type, data_value) - self.assertIsNotNone(self.metadata_data['media_url']) + self.assertIsNotNone(self.metadata_data["media_url"]) - request = self.factory.get('/', **self.extra) - ext = self.data_value[self.data_value.rindex('.') + 1:] + request = self.factory.get("/", **self.extra) + ext = self.data_value[self.data_value.rindex(".") + 1 :] response = self.view(request, pk=self.metadata.pk, format=ext) self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], - 'attachment; filename=transportation.csv') + self.assertEqual( + response["Content-Disposition"], "attachment; filename=transportation.csv" + ) def test_add_media_geojson_link(self): - data_type = 'media' - data_value = 'xform_geojson {} transportation'.format(self.xform.pk) + data_type = "media" + data_value = "xform_geojson {} transportation".format(self.xform.pk) extra_data = { "data_title": "test", "data_simple_style": True, "data_geo_field": "test", - "data_fields": "transport/available_transportation_types_to_referral_facility/ambulance" # noqa + "data_fields": "transport/available_transportation_types_to_referral_facility/ambulance", # noqa } self._add_form_metadata( - self.xform, - data_type, - data_value, - extra_data=extra_data + self.xform, data_type, data_value, extra_data=extra_data ) - self.assertIsNotNone(self.metadata_data['media_url']) - request = self.factory.get('/', **self.extra) - ext = self.data_value[self.data_value.rindex('.') + 1:] + self.assertIsNotNone(self.metadata_data["media_url"]) + request = self.factory.get("/", **self.extra) + ext = self.data_value[self.data_value.rindex(".") + 1 :] response = self.view(request, pk=self.metadata.pk, format=ext) self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], - 'attachment; filename=transportation.geojson') + self.assertEqual( + response["Content-Disposition"], + "attachment; filename=transportation.geojson", + ) def test_add_media_dataview_link(self): self._create_dataview() - data_type = 'media' - data_value = 'dataview {} transportation'.format(self.data_view.pk) + data_type = "media" + data_value = "dataview {} transportation".format(self.data_view.pk) self._add_form_metadata(self.xform, data_type, data_value) - self.assertIsNotNone(self.metadata_data['media_url']) + self.assertIsNotNone(self.metadata_data["media_url"]) - request = self.factory.get('/', **self.extra) - ext = self.data_value[self.data_value.rindex('.') + 1:] + request = self.factory.get("/", **self.extra) + ext = self.data_value[self.data_value.rindex(".") + 1 :] response = self.view(request, pk=self.metadata.pk, format=ext) self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], - 'attachment; filename=transportation.csv') + self.assertEqual( + response["Content-Disposition"], "attachment; filename=transportation.csv" + ) def test_invalid_post(self): response = self._post_metadata({}, False) self.assertEqual(response.status_code, 400) - response = self._post_metadata({ - 'data_type': 'supporting_doc'}, False) + response = self._post_metadata({"data_type": "supporting_doc"}, False) self.assertEqual(response.status_code, 400) - response = self._post_metadata({ - 'data_type': 'supporting_doc', - 'xform': self.xform.pk - }, False) + response = self._post_metadata( + {"data_type": "supporting_doc", "xform": self.xform.pk}, False + ) self.assertEqual(response.status_code, 400) - response = self._post_metadata({ - 'data_type': 'supporting_doc', - 'data_value': 'supporting.doc' - }, False) + response = self._post_metadata( + {"data_type": "supporting_doc", "data_value": "supporting.doc"}, False + ) self.assertEqual(response.status_code, 400) def _add_test_metadata(self): - for data_type in ['supporting_doc', 'media', 'source']: - self._add_form_metadata(self.xform, data_type, - self.data_value, self.path) + for data_type in ["supporting_doc", "media", "source"]: + self._add_form_metadata(self.xform, data_type, self.data_value, self.path) def test_list_metadata(self): self._add_test_metadata() - self.view = MetaDataViewSet.as_view({'get': 'list'}) - request = self.factory.get('/') + self.view = MetaDataViewSet.as_view({"get": "list"}) + request = self.factory.get("/") response = self.view(request) self.assertEqual(response.status_code, 401) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) def test_list_metadata_for_specific_form(self): self._add_test_metadata() - self.view = MetaDataViewSet.as_view({'get': 'list'}) - data = {'xform': self.xform.pk} - request = self.factory.get('/', data) + self.view = MetaDataViewSet.as_view({"get": "list"}) + data = {"xform": self.xform.pk} + request = self.factory.get("/", data) response = self.view(request) self.assertEqual(response.status_code, 401) - request = self.factory.get('/', data, **self.extra) + request = self.factory.get("/", data, **self.extra) response = self.view(request) - self.assertNotEqual(response.get('Cache-Control'), None) + self.assertNotEqual(response.get("Cache-Control"), None) self.assertEqual(response.status_code, 200) - data['xform'] = 1234509909 - request = self.factory.get('/', data, **self.extra) + data["xform"] = 1234509909 + request = self.factory.get("/", data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 404) - data['xform'] = "INVALID" - request = self.factory.get('/', data, **self.extra) + data["xform"] = "INVALID" + request = self.factory.get("/", data, **self.extra) response = self.view(request) self.assertEqual(response.status_code, 400) def test_project_metadata_has_project_field(self): self._add_project_metadata( - self.project, 'supporting_doc', self.data_value, self.path) + self.project, "supporting_doc", self.data_value, self.path + ) # Test json of project metadata - request = self.factory.get('/', **self.extra) - response = self.view(request, pk=self.metadata_data['id']) + request = self.factory.get("/", **self.extra) + response = self.view(request, pk=self.metadata_data["id"]) self.assertEqual(response.status_code, 200) data = dict(response.data) - self.assertIsNotNone(data['media_url']) - self.assertEqual(data['project'], self.metadata.object_id) + self.assertIsNotNone(data["media_url"]) + self.assertEqual(data["project"], self.metadata.object_id) def test_instance_metadata_has_instance_field(self): - self._add_instance_metadata( - 'supporting_doc', self.data_value, self.path) + self._add_instance_metadata("supporting_doc", self.data_value, self.path) # Test json of project metadata - request = self.factory.get('/', **self.extra) - response = self.view(request, pk=self.metadata_data['id']) + request = self.factory.get("/", **self.extra) + response = self.view(request, pk=self.metadata_data["id"]) self.assertEqual(response.status_code, 200) data = dict(response.data) - self.assertIsNotNone(data['media_url']) - self.assertEqual(data['instance'], self.metadata.object_id) + self.assertIsNotNone(data["media_url"]) + self.assertEqual(data["instance"], self.metadata.object_id) def test_should_return_both_xform_and_project_metadata(self): # delete all existing metadata @@ -427,14 +436,16 @@ def test_should_return_both_xform_and_project_metadata(self): expected_metadata_count = 2 project_response = self._add_project_metadata( - self.project, 'media', "check.png", self.path) - self.assertTrue("image/png" in project_response.data['data_file_type']) + self.project, "media", "check.png", self.path + ) + self.assertTrue("image/png" in project_response.data["data_file_type"]) form_response = self._add_form_metadata( - self.xform, 'supporting_doc', "bla.png", self.path) - self.assertTrue("image/png" in form_response.data['data_file_type']) + self.xform, "supporting_doc", "bla.png", self.path + ) + self.assertTrue("image/png" in form_response.data["data_file_type"]) - view = MetaDataViewSet.as_view({'get': 'list'}) + view = MetaDataViewSet.as_view({"get": "list"}) request = self.factory.get("/", **self.extra) response = view(request) @@ -442,23 +453,21 @@ def test_should_return_both_xform_and_project_metadata(self): for record in response.data: if record.get("xform"): - self.assertEqual(record.get('xform'), self.xform.id) - self.assertIsNone(record.get('project')) + self.assertEqual(record.get("xform"), self.xform.id) + self.assertIsNone(record.get("project")) else: - self.assertEqual(record.get('project'), self.project.id) - self.assertIsNone(record.get('xform')) + self.assertEqual(record.get("project"), self.project.id) + self.assertIsNone(record.get("xform")) def test_should_only_return_xform_metadata(self): # delete all existing metadata MetaData.objects.all().delete() - self._add_project_metadata( - self.project, 'media', "check.png", self.path) + self._add_project_metadata(self.project, "media", "check.png", self.path) - self._add_form_metadata( - self.xform, 'supporting_doc', "bla.png", self.path) + self._add_form_metadata(self.xform, "supporting_doc", "bla.png", self.path) - view = MetaDataViewSet.as_view({'get': 'list'}) + view = MetaDataViewSet.as_view({"get": "list"}) query_data = {"xform": self.xform.id} request = self.factory.get("/", data=query_data, **self.extra) response = view(request) @@ -468,15 +477,15 @@ def test_should_only_return_xform_metadata(self): self.assertNotIn("project", response.data[0]) def _create_metadata_object(self): - view = MetaDataViewSet.as_view({'post': 'create'}) - with open(self.path, 'rb') as media_file: + view = MetaDataViewSet.as_view({"post": "create"}) + with open(self.path, "rb") as media_file: data = { - 'data_type': 'media', - 'data_value': 'check.png', - 'data_file': media_file, - 'project': self.project.id + "data_type": "media", + "data_value": "check.png", + "data_file": media_file, + "project": self.project.id, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) return response @@ -492,134 +501,123 @@ def test_integrity_error_is_handled(self): self.assertEqual(response.status_code, 400) def test_invalid_form_metadata(self): - view = MetaDataViewSet.as_view({'post': 'create'}) - with open(self.path, 'rb') as media_file: + view = MetaDataViewSet.as_view({"post": "create"}) + with open(self.path, "rb") as media_file: data = { - 'data_type': "media", - 'data_value': self.data_value, - 'xform': 999912, - 'data_file': media_file, + "data_type": "media", + "data_value": self.data_value, + "xform": 999912, + "data_file": media_file, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) - self.assertEqual(response.data, - {'xform': ['XForm does not exist']}) + self.assertEqual(response.data, {"xform": ["XForm does not exist"]}) def test_xform_meta_permission(self): - view = MetaDataViewSet.as_view({'post': 'create'}) + view = MetaDataViewSet.as_view({"post": "create"}) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor-minor|dataentry', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor-minor|dataentry", + "xform": self.xform.pk, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) meta = MetaData.xform_meta_permission(self.xform) - self.assertEqual(meta.data_value, response.data.get('data_value')) + self.assertEqual(meta.data_value, response.data.get("data_value")) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor-minors|invalid_role', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor-minors|invalid_role", + "xform": self.xform.pk, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 400) - error = u"Format 'role'|'role' or Invalid role" - self.assertEqual(response.data, {'non_field_errors': [error]}) + error = "Format 'role'|'role' or Invalid role" + self.assertEqual(response.data, {"non_field_errors": [error]}) def test_role_update_xform_meta_perms(self): - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} alice_profile = self._create_user_profile(alice_data) EditorRole.add(alice_profile.user, self.xform) - view = MetaDataViewSet.as_view({ - 'post': 'create', - 'put': 'update' - }) + view = MetaDataViewSet.as_view({"post": "create", "put": "update"}) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor-minor|dataentry', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor-minor|dataentry", + "xform": self.xform.pk, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) - self.assertFalse( - EditorRole.user_has_role(alice_profile.user, self.xform)) + self.assertFalse(EditorRole.user_has_role(alice_profile.user, self.xform)) - self.assertTrue( - EditorMinorRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(EditorMinorRole.user_has_role(alice_profile.user, self.xform)) meta = MetaData.xform_meta_permission(self.xform) DataEntryRole.add(alice_profile.user, self.xform) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor|dataentry-only', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor|dataentry-only", + "xform": self.xform.pk, } - request = self.factory.put('/', data, **self.extra) + request = self.factory.put("/", data, **self.extra) response = view(request, pk=meta.pk) self.assertEqual(response.status_code, 200) - self.assertFalse( - DataEntryRole.user_has_role(alice_profile.user, self.xform)) + self.assertFalse(DataEntryRole.user_has_role(alice_profile.user, self.xform)) - self.assertTrue( - DataEntryOnlyRole.user_has_role(alice_profile.user, self.xform)) + self.assertTrue(DataEntryOnlyRole.user_has_role(alice_profile.user, self.xform)) def test_xform_meta_perms_duplicates(self): - view = MetaDataViewSet.as_view({ - 'post': 'create', - 'put': 'update' - }) + view = MetaDataViewSet.as_view({"post": "create", "put": "update"}) ct = ContentType.objects.get_for_model(self.xform) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor-minor|dataentry', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor-minor|dataentry", + "xform": self.xform.pk, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) - count = MetaData.objects.filter(data_type=XFORM_META_PERMS, - object_id=self.xform.pk, - content_type=ct.pk).count() + count = MetaData.objects.filter( + data_type=XFORM_META_PERMS, object_id=self.xform.pk, content_type=ct.pk + ).count() self.assertEqual(1, count) data = { - 'data_type': XFORM_META_PERMS, - 'data_value': 'editor-minor|dataentry-only', - 'xform': self.xform.pk + "data_type": XFORM_META_PERMS, + "data_value": "editor-minor|dataentry-only", + "xform": self.xform.pk, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) response = view(request) self.assertEqual(response.status_code, 201) - count = MetaData.objects.filter(data_type=XFORM_META_PERMS, - object_id=self.xform.pk, - content_type=ct.pk).count() + count = MetaData.objects.filter( + data_type=XFORM_META_PERMS, object_id=self.xform.pk, content_type=ct.pk + ).count() self.assertEqual(1, count) @@ -634,13 +632,13 @@ def test_unique_submission_review_metadata(self): response = self._add_form_metadata(self.xform, data_type, data_value) # Duplicate with different Data Value - view = MetaDataViewSet.as_view({'post': 'create'}) + view = MetaDataViewSet.as_view({"post": "create"}) data = { - 'xform': response.data['xform'], - 'data_type': data_type, - 'data_value': False, + "xform": response.data["xform"], + "data_type": data_type, + "data_value": False, } - request = self.factory.post('/', data, **self.extra) + request = self.factory.post("/", data, **self.extra) d_response = view(request) self.assertEqual(d_response.status_code, 400) diff --git a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py index 7ae8ae6f09..3425b6fe23 100644 --- a/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_organization_profile_viewset.py @@ -4,13 +4,13 @@ """ import json from builtins import str as text +from unittest.mock import patch from django.contrib.auth.models import User, timezone from django.core.cache import cache from django.test.utils import override_settings from guardian.shortcuts import get_perms -from mock import patch from rest_framework import status from onadata.apps.api.models.organization_profile import ( diff --git a/onadata/apps/api/tests/viewsets/test_osm_viewset.py b/onadata/apps/api/tests/viewsets/test_osm_viewset.py index 4ba649601c..c2b5c8618a 100644 --- a/onadata/apps/api/tests/viewsets/test_osm_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_osm_viewset.py @@ -1,6 +1,11 @@ +# -*- coding: utf-8 -*- +""" +Test OSMViewSet module. +""" import csv import os from io import StringIO +from unittest.mock import patch from django.conf import settings from django.db import IntegrityError, transaction @@ -9,8 +14,6 @@ from django.test.utils import override_settings from django.utils.dateparse import parse_datetime -from mock import patch - from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.data_viewset import DataViewSet from onadata.apps.api.viewsets.osm_viewset import OsmViewSet @@ -25,6 +28,10 @@ class TestOSMViewSet(TestAbstractViewSet): + """ + Test OSMViewSet module. + """ + def setUp(self): super(self.__class__, self).setUp() self._login_user_and_profile() diff --git a/onadata/apps/api/tests/viewsets/test_project_viewset.py b/onadata/apps/api/tests/viewsets/test_project_viewset.py index 2a787b6f6c..1695ac9360 100644 --- a/onadata/apps/api/tests/viewsets/test_project_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_project_viewset.py @@ -7,6 +7,7 @@ from collections import OrderedDict from datetime import datetime from operator import itemgetter +from unittest.mock import MagicMock, Mock, patch from django.conf import settings from django.contrib.auth import get_user_model @@ -18,7 +19,6 @@ import dateutil.parser import requests from httmock import HTTMock, urlmatch -from mock import MagicMock, Mock, patch from rest_framework.authtoken.models import Token from six import iteritems diff --git a/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py b/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py index 24b3bb729a..c442439530 100644 --- a/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py @@ -3,15 +3,16 @@ """ from __future__ import unicode_literals -from mock import patch +from unittest.mock import patch + from rest_framework.test import APIRequestFactory from onadata.apps.api.viewsets.submission_review_viewset import SubmissionReviewViewSet -from onadata.apps.logger.models import SubmissionReview, Instance +from onadata.apps.logger.models import Instance, SubmissionReview from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.messaging.constants import XFORM, SUBMISSION_REVIEWED -from onadata.libs.permissions import EditorRole, OwnerRole, ManagerRole -from onadata.libs.utils.common_tags import REVIEW_STATUS, REVIEW_COMMENT +from onadata.apps.messaging.constants import SUBMISSION_REVIEWED, XFORM +from onadata.libs.permissions import EditorRole, ManagerRole, OwnerRole +from onadata.libs.utils.common_tags import REVIEW_COMMENT, REVIEW_STATUS class TestSubmissionReviewViewSet(TestBase): @@ -65,7 +66,7 @@ def test_submission_review_create(self, mock_send_message): ) # sends message upon saving the submission review self.assertTrue(mock_send_message.called) - mock_send_message.called_with( + mock_send_message.assert_called_with( submission_review.id, submission_review.instance.xform.id, XFORM, @@ -100,7 +101,7 @@ def test_bulk_create_submission_review(self, mock_send_message): already_seen = [] # sends message upon saving the submission review self.assertTrue(mock_send_message.called) - mock_send_message.called_with( + mock_send_message.assert_called_with( [s.id for s in self.xform.instances.all()], self.xform.id, XFORM, diff --git a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py index aaed53b8d5..cdcef75cd1 100644 --- a/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_user_profile_viewset.py @@ -6,21 +6,21 @@ import datetime import json import os -from six.moves.urllib.parse import urlparse, parse_qs +from unittest.mock import call, patch from django.contrib.auth import get_user_model from django.core.cache import cache from django.db.models import signals from django.test.utils import override_settings -from django.utils.dateparse import parse_datetime from django.utils import timezone +from django.utils.dateparse import parse_datetime import requests from django_digest.test import DigestAuth from httmock import HTTMock, all_requests -from mock import patch, call from registration.models import RegistrationProfile from rest_framework.authtoken.models import Token +from six.moves.urllib.parse import parse_qs, urlparse from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.connect_viewset import ConnectViewSet @@ -30,9 +30,8 @@ from onadata.apps.main.models import UserProfile from onadata.apps.main.models.user_profile import set_kpi_formbuilder_permissions from onadata.libs.authentication import DigestAuthentication -from onadata.libs.serializers.user_profile_serializer import _get_first_last_names from onadata.libs.permissions import EditorRole - +from onadata.libs.serializers.user_profile_serializer import _get_first_last_names User = get_user_model() diff --git a/onadata/apps/api/tests/viewsets/test_xform_list_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_list_viewset.py index 6f4709c2ea..9869903b09 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_list_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_list_viewset.py @@ -1,13 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Test XFormListViewSet module. +""" import os from builtins import open from hashlib import md5 +from unittest.mock import patch from django.conf import settings from django.test import TransactionTestCase from django.urls import reverse + from django_digest.test import Client as DigestClient from django_digest.test import DigestAuth -from mock import patch from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.api.viewsets.project_viewset import ProjectViewSet @@ -16,10 +21,14 @@ XFormListViewSet, ) from onadata.apps.main.models import MetaData -from onadata.libs.permissions import DataEntryRole, ReadOnlyRole, OwnerRole +from onadata.libs.permissions import DataEntryRole, OwnerRole, ReadOnlyRole class TestXFormListViewSet(TestAbstractViewSet, TransactionTestCase): + """ + Test XFormListViewSet module. + """ + def setUp(self): super(self.__class__, self).setUp() self.view = XFormListViewSet.as_view({"get": "list"}) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 049a0b12cd..9b9c385839 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -14,6 +14,7 @@ from datetime import datetime, timedelta from http.client import BadStatusLine from io import StringIO +from unittest.mock import Mock, patch from xml.dom import Node, minidom from django.conf import settings @@ -31,7 +32,6 @@ from django_digest.test import DigestAuth from flaky import flaky from httmock import HTTMock -from mock import Mock, patch from rest_framework import status from onadata.apps.api.tests.mocked_data import ( @@ -214,7 +214,7 @@ def test_replace_form_with_external_choices(self, mock_send_message): self.assertEqual(response.status_code, 200) # send message upon form update self.assertTrue(mock_send_message.called) - mock_send_message.called_with( + mock_send_message.assert_called_with( self.xform.id, self.xform.id, XFORM, request.user, FORM_UPDATED ) diff --git a/onadata/apps/logger/tests/models/test_instance.py b/onadata/apps/logger/tests/models/test_instance.py index 89222d97bc..5c2d9db30f 100644 --- a/onadata/apps/logger/tests/models/test_instance.py +++ b/onadata/apps/logger/tests/models/test_instance.py @@ -4,13 +4,13 @@ """ import os from datetime import datetime, timedelta +from unittest.mock import Mock, patch from django.http.request import HttpRequest from django.test import override_settings from django.utils.timezone import utc from django_digest.test import DigestAuth -from mock import Mock, patch from onadata.apps.logger.models import Instance, SubmissionReview, XForm from onadata.apps.logger.models.instance import ( diff --git a/onadata/apps/logger/tests/test_form_submission.py b/onadata/apps/logger/tests/test_form_submission.py index 4ab3d651dc..16c2d6b7ad 100644 --- a/onadata/apps/logger/tests/test_form_submission.py +++ b/onadata/apps/logger/tests/test_form_submission.py @@ -4,25 +4,25 @@ """ import os import re - from contextlib import contextmanager -from django.http import Http404 -from django.http import UnreadablePostError -from django_digest.test import DigestAuth -from django_digest.test import Client as DigestClient +from unittest.mock import ANY, Mock, patch + +from django.http import Http404, UnreadablePostError from django.test.utils import override_settings + +from django_digest.test import Client as DigestClient +from django_digest.test import DigestAuth from guardian.shortcuts import assign_perm -from mock import patch, Mock, ANY from nose import SkipTest -from onadata.apps.main.models.user_profile import UserProfile -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models import Instance from onadata.apps.logger.models.instance import InstanceHistory from onadata.apps.logger.models.project import Project from onadata.apps.logger.models.xform import XForm from onadata.apps.logger.xform_instance_parser import clean_and_parse_xml -from onadata.apps.viewer.models.parsed_instance import query_data, query_count +from onadata.apps.main.models.user_profile import UserProfile +from onadata.apps.main.tests.test_base import TestBase +from onadata.apps.viewer.models.parsed_instance import query_count, query_data from onadata.apps.viewer.signals import process_submission from onadata.libs.utils.common_tags import GEOLOCATION, LAST_EDITED diff --git a/onadata/apps/main/tests/test_process.py b/onadata/apps/main/tests/test_process.py index 113e575403..0dc53e8f6b 100644 --- a/onadata/apps/main/tests/test_process.py +++ b/onadata/apps/main/tests/test_process.py @@ -10,6 +10,7 @@ from datetime import datetime from hashlib import md5 from io import BytesIO +from unittest.mock import patch from xml.dom import Node, minidom from django.conf import settings @@ -22,7 +23,6 @@ import requests from django_digest.test import Client as DigestClient from flaky import flaky -from mock import patch from six import iteritems from onadata.apps.logger.models import XForm diff --git a/onadata/apps/main/tests/test_user_profile.py b/onadata/apps/main/tests/test_user_profile.py index 0bc68bbd81..f83225c916 100644 --- a/onadata/apps/main/tests/test_user_profile.py +++ b/onadata/apps/main/tests/test_user_profile.py @@ -1,19 +1,25 @@ +# -*- coding: utf-8 -*- +""" +Test user profile +""" from __future__ import unicode_literals -from django.contrib.auth.models import AnonymousUser -from django.contrib.auth.models import User -from django.test import RequestFactory -from django.test import TestCase +from unittest.mock import patch + +from django.contrib.auth.models import AnonymousUser, User +from django.test import RequestFactory, TestCase from django.test.client import Client from django.urls import reverse -from mock import patch from onadata.apps.logger.xform_instance_parser import XLSFormError -from onadata.apps.main.views import profile, api_token +from onadata.apps.main.views import api_token, profile from onadata.libs.utils.common_tools import merge_dicts class TestUserProfile(TestCase): + """ + Test user profile + """ def setup(self): self.client = Client() @@ -21,82 +27,78 @@ def setup(self): def _login_user_and_profile(self, extra_post_data={}): post_data = { - 'username': 'bob', - 'email': 'bob@columbia.edu', - 'password1': 'bobbob', - 'password2': 'bobbob', - 'first_name': 'Bob', - 'last_name': 'User', - 'city': 'Bobville', - 'country': 'US', - 'organization': 'Bob Inc.', - 'home_page': 'bob.com', - 'twitter': 'boberama' + "username": "bob", + "email": "bob@columbia.edu", + "password1": "bobbob", + "password2": "bobbob", + "first_name": "Bob", + "last_name": "User", + "city": "Bobville", + "country": "US", + "organization": "Bob Inc.", + "home_page": "bob.com", + "twitter": "boberama", } - url = '/accounts/register/' + url = "/accounts/register/" post_data = merge_dicts(post_data, extra_post_data) self.response = self.client.post(url, post_data) try: - self.user = User.objects.get(username=post_data['username']) + self.user = User.objects.get(username=post_data["username"]) except User.DoesNotExist: pass def test_create_user_with_given_name(self): self._login_user_and_profile() self.assertEqual(self.response.status_code, 302) - self.assertEqual(self.user.username, 'bob') + self.assertEqual(self.user.username, "bob") - @patch('onadata.apps.main.views.render') + @patch("onadata.apps.main.views.render") def test_xlsform_error_returns_400(self, mock_render): - mock_render.side_effect = XLSFormError( - "Title shouldn't have an ampersand") + mock_render.side_effect = XLSFormError("Title shouldn't have an ampersand") self._login_user_and_profile() - response = self.client.get( - reverse(profile, kwargs={ - 'username': "bob" - })) + response = self.client.get(reverse(profile, kwargs={"username": "bob"})) self.assertTrue(mock_render.called) self.assertEqual(response.status_code, 400) - self.assertEqual(response.content.decode('utf-8'), - "Title shouldn't have an ampersand") + self.assertEqual( + response.content.decode("utf-8"), "Title shouldn't have an ampersand" + ) def test_create_user_profile_for_user(self): self._login_user_and_profile() self.assertEqual(self.response.status_code, 302) user_profile = self.user.profile - self.assertEqual(user_profile.city, 'Bobville') - self.assertTrue(hasattr(user_profile, 'metadata')) + self.assertEqual(user_profile.city, "Bobville") + self.assertTrue(hasattr(user_profile, "metadata")) def test_disallow_non_alpha_numeric(self): invalid_usernames = [ - 'b ob', - 'b.o.b.', - 'b-ob', - 'b!', - '@bob', - 'bob@bob.com', - 'bob$', - 'b&o&b', - 'bob?', - '#bob', - '(bob)', - 'b*ob', - '%s % bob', + "b ob", + "b.o.b.", + "b-ob", + "b!", + "@bob", + "bob@bob.com", + "bob$", + "b&o&b", + "bob?", + "#bob", + "(bob)", + "b*ob", + "%s % bob", ] users_before = User.objects.count() for username in invalid_usernames: - self._login_user_and_profile({'username': username}) + self._login_user_and_profile({"username": username}) self.assertEqual(User.objects.count(), users_before) def test_disallow_reserved_name(self): users_before = User.objects.count() - self._login_user_and_profile({'username': 'admin'}) + self._login_user_and_profile({"username": "admin"}) self.assertEqual(User.objects.count(), users_before) def test_404_if_user_does_not_exist(self): - response = self.client.get(reverse(profile, - kwargs={'username': 'nonuser'})) + response = self.client.get(reverse(profile, kwargs={"username": "nonuser"})) self.assertEqual(response.status_code, 404) def test_403_if_unauthorised_user_tries_to_access_api_token_link(self): @@ -105,54 +107,48 @@ def test_403_if_unauthorised_user_tries_to_access_api_token_link(self): # create user alice post_data = { - 'username': 'alice', - 'email': 'alice@columbia.edu', - 'password1': 'alicealice', - 'password2': 'alicealice', - 'first_name': 'Alice', - 'last_name': 'Wonderland', - 'city': 'Aliceville', - 'country': 'KE', - 'organization': 'Alice Inc.', - 'home_page': 'alice.com', - 'twitter': 'alicemsweet' + "username": "alice", + "email": "alice@columbia.edu", + "password1": "alicealice", + "password2": "alicealice", + "first_name": "Alice", + "last_name": "Wonderland", + "city": "Aliceville", + "country": "KE", + "organization": "Alice Inc.", + "home_page": "alice.com", + "twitter": "alicemsweet", } - url = '/accounts/register/' + url = "/accounts/register/" self.client.post(url, post_data) # try accessing api-token with an anonymous user - request = factory.get('/api-token') + request = factory.get("/api-token") request.user = AnonymousUser() - response = api_token(request, 'alice') + response = api_token(request, "alice") self.assertEqual(response.status_code, 302) # login with user bob self._login_user_and_profile() # try accessing api-token with user 'bob' but with username 'alice' - request = factory.get('/api-token') + request = factory.get("/api-token") request.user = self.user - response = api_token(request, 'alice') + response = api_token(request, "alice") self.assertEqual(response.status_code, 403) # try accessing api-token with user 'bob' but with username 'bob' - request = factory.get('/api-token') + request = factory.get("/api-token") request.user = self.user response = api_token(request, self.user.username) self.assertEqual(response.status_code, 200) def test_show_single_at_sign_in_twitter_link(self): self._login_user_and_profile() - response = self.client.get( - reverse(profile, kwargs={ - 'username': "bob" - })) + response = self.client.get(reverse(profile, kwargs={"username": "bob"})) self.assertContains(response, ">@boberama") # add the @ sign self.user.profile.twitter = "@boberama" self.user.profile.save() - response = self.client.get( - reverse(profile, kwargs={ - 'username': "bob" - })) + response = self.client.get(reverse(profile, kwargs={"username": "bob"})) self.assertContains(response, ">@boberama") diff --git a/onadata/apps/messaging/tests/test_backends_mqtt.py b/onadata/apps/messaging/tests/test_backends_mqtt.py index e4d3d3176e..f9bcd0f925 100644 --- a/onadata/apps/messaging/tests/test_backends_mqtt.py +++ b/onadata/apps/messaging/tests/test_backends_mqtt.py @@ -6,36 +6,37 @@ import json import ssl +from unittest.mock import MagicMock, patch from django.test import TestCase -from mock import MagicMock, patch - -from onadata.apps.messaging.backends.mqtt import (MQTTBackend, get_payload, - get_target_metadata) +from onadata.apps.messaging.backends.mqtt import ( + MQTTBackend, + get_payload, + get_target_metadata, +) from onadata.apps.messaging.constants import PROJECT, XFORM -from onadata.apps.messaging.tests.test_base import (_create_message, - _create_user) +from onadata.apps.messaging.tests.test_base import _create_message, _create_user class TestMQTTBackend(TestCase): """ Test MQTT Backend """ + maxDiff = None def test_mqtt_get_topic(self): """ Test MQTT backend get_topic method """ - from_user = _create_user('Bob') - to_user = _create_user('Alice') - instance = _create_message(from_user, to_user, 'I love oov') - mqtt = MQTTBackend(options={'HOST': 'localhost'}) - expected = ( - "/{topic_root}/{target_name}/{target_id}/messages/publish".format( - topic_root='onadata', target_name='user', - target_id=to_user.id)) + from_user = _create_user("Bob") + to_user = _create_user("Alice") + instance = _create_message(from_user, to_user, "I love oov") + mqtt = MQTTBackend(options={"HOST": "localhost"}) + expected = "/{topic_root}/{target_name}/{target_id}/messages/publish".format( + topic_root="onadata", target_name="user", target_id=to_user.id + ) self.assertEqual(expected, mqtt.get_topic(instance)) def test_get_target_metadata(self): @@ -44,106 +45,107 @@ def test_get_target_metadata(self): """ # User objects - user = _create_user('John') - user_metadata = {'id': user.pk, 'name': user.get_full_name()} + user = _create_user("John") + user_metadata = {"id": user.pk, "name": user.get_full_name()} self.assertEqual( - json.dumps(user_metadata), json.dumps(get_target_metadata(user))) + json.dumps(user_metadata), json.dumps(get_target_metadata(user)) + ) # XForm objects xform = MagicMock() xform.pk = 1337 - xform.title = 'Test Form' - xform.id_string = 'Test_Form_ID' + xform.title = "Test Form" + xform.id_string = "Test_Form_ID" xform._meta.model_name = XFORM - xform_metadata = { - 'id': 1337, - 'name': 'Test Form', - 'form_id': 'Test_Form_ID' - } + xform_metadata = {"id": 1337, "name": "Test Form", "form_id": "Test_Form_ID"} self.assertEqual( - json.dumps(xform_metadata), json.dumps(get_target_metadata(xform))) + json.dumps(xform_metadata), json.dumps(get_target_metadata(xform)) + ) # Project objects project = MagicMock() project.pk = 7331 - project.name = 'Test Project' + project.name = "Test Project" project._meta.model_name = PROJECT - project_metadata = {'id': 7331, 'name': 'Test Project'} + project_metadata = {"id": 7331, "name": "Test Project"} self.assertEqual( - json.dumps(project_metadata), - json.dumps(get_target_metadata(project))) + json.dumps(project_metadata), json.dumps(get_target_metadata(project)) + ) def test_mqtt_get_payload(self): """ Test MQTT backend get_payload function """ - from_user = _create_user('Bob') - to_user = _create_user('Alice') - instance = _create_message(from_user, to_user, 'I love oov') + from_user = _create_user("Bob") + to_user = _create_user("Alice") + instance = _create_message(from_user, to_user, "I love oov") payload = { - 'id': instance.id, - 'time': instance.timestamp.isoformat(), - 'payload': { - 'author': { - 'username': from_user.username, - 'real_name': from_user.get_full_name() + "id": instance.id, + "time": instance.timestamp.isoformat(), + "payload": { + "author": { + "username": from_user.username, + "real_name": from_user.get_full_name(), }, - 'context': { - 'type': to_user._meta.model_name, - 'metadata': { - 'id': to_user.pk, - 'name': to_user.get_full_name() - }, - 'verb': 'message' + "context": { + "type": to_user._meta.model_name, + "metadata": {"id": to_user.pk, "name": to_user.get_full_name()}, + "verb": "message", }, - 'message': "I love oov" - } + "message": "I love oov", + }, } self.assertEqual( - json.dumps(payload), get_payload(instance, verbose_payload=True)) + json.dumps(payload), get_payload(instance, verbose_payload=True) + ) expected_payload = { - 'id': instance.id, - 'verb': instance.verb, - 'message': "I love oov", - 'user': from_user.username, - 'timestamp': instance.timestamp.isoformat() + "id": instance.id, + "verb": instance.verb, + "message": "I love oov", + "user": from_user.username, + "timestamp": instance.timestamp.isoformat(), } self.assertEqual( - json.dumps(expected_payload), get_payload( - instance, verbose_payload=False)) + json.dumps(expected_payload), get_payload(instance, verbose_payload=False) + ) - @patch('onadata.apps.messaging.backends.mqtt.publish.single') + @patch("onadata.apps.messaging.backends.mqtt.publish.single") def test_mqtt_send(self, mocked): """ Test MQTT Backend send method """ - from_user = _create_user('Bob') - to_user = _create_user('Alice') - instance = _create_message(from_user, to_user, 'I love oov') - mqtt = MQTTBackend(options={ - 'HOST': 'localhost', - 'PORT': 8883, - 'SECURE': True, - 'CA_CERT_FILE': 'cacert.pem', - 'CERT_FILE': 'emq.pem', - 'KEY_FILE': 'emq.key' - }) + from_user = _create_user("Bob") + to_user = _create_user("Alice") + instance = _create_message(from_user, to_user, "I love oov") + mqtt = MQTTBackend( + options={ + "HOST": "localhost", + "PORT": 8883, + "SECURE": True, + "CA_CERT_FILE": "cacert.pem", + "CERT_FILE": "emq.pem", + "KEY_FILE": "emq.key", + } + ) mqtt.send(instance=instance) self.assertTrue(mocked.called) args, kwargs = mocked.call_args_list[0] self.assertEqual(mqtt.get_topic(instance), args[0]) - self.assertEqual(get_payload(instance), kwargs['payload']) - self.assertEqual('localhost', kwargs['hostname']) - self.assertEqual(8883, kwargs['port']) - self.assertEqual(0, kwargs['qos']) - self.assertEqual(False, kwargs['retain']) + self.assertEqual(get_payload(instance), kwargs["payload"]) + self.assertEqual("localhost", kwargs["hostname"]) + self.assertEqual(8883, kwargs["port"]) + self.assertEqual(0, kwargs["qos"]) + self.assertEqual(False, kwargs["retain"]) self.assertDictEqual( - dict(ca_certs='cacert.pem', - certfile='emq.pem', - keyfile='emq.key', - tls_version=ssl.PROTOCOL_TLSv1_2, - cert_reqs=ssl.CERT_NONE), - kwargs['tls']) + dict( + ca_certs="cacert.pem", + certfile="emq.pem", + keyfile="emq.key", + tls_version=ssl.PROTOCOL_TLSv1_2, + cert_reqs=ssl.CERT_NONE, + ), + kwargs["tls"], + ) diff --git a/onadata/apps/messaging/tests/test_signals.py b/onadata/apps/messaging/tests/test_signals.py index f05f9b42d7..0342a58ec8 100644 --- a/onadata/apps/messaging/tests/test_signals.py +++ b/onadata/apps/messaging/tests/test_signals.py @@ -4,10 +4,12 @@ """ from __future__ import unicode_literals -from actstream.models import Action +from unittest.mock import patch + from django.test import TestCase from django.test.utils import override_settings -from mock import patch + +from actstream.models import Action from onadata.apps.messaging.signals import messaging_backends_handler @@ -20,12 +22,11 @@ class TestSignals(TestCase): # pylint: disable=invalid-name @override_settings( NOTIFICATION_BACKENDS={ - 'mqtt': { - 'BACKEND': 'onadata.apps.messaging.backends.base.BaseBackend' - }, + "mqtt": {"BACKEND": "onadata.apps.messaging.backends.base.BaseBackend"}, }, - MESSAGING_ASYNC_NOTIFICATION=True) - @patch('onadata.apps.messaging.signals.call_backend_async.apply_async') + MESSAGING_ASYNC_NOTIFICATION=True, + ) + @patch("onadata.apps.messaging.signals.call_backend_async.apply_async") def test_messaging_backends_handler_async(self, call_backend_async_mock): """ Test messaging backends handler function. @@ -33,15 +34,15 @@ def test_messaging_backends_handler_async(self, call_backend_async_mock): messaging_backends_handler(Action, instance=Action(id=9), created=True) self.assertTrue(call_backend_async_mock.called) call_backend_async_mock.assert_called_with( - ('onadata.apps.messaging.backends.base.BaseBackend', 9, None), - countdown=2) + ("onadata.apps.messaging.backends.base.BaseBackend", 9, None), countdown=2 + ) - @override_settings(NOTIFICATION_BACKENDS={ - 'mqtt': { - 'BACKEND': 'onadata.apps.messaging.backends.base.BaseBackend' - }, - }) - @patch('onadata.apps.messaging.signals.call_backend') + @override_settings( + NOTIFICATION_BACKENDS={ + "mqtt": {"BACKEND": "onadata.apps.messaging.backends.base.BaseBackend"}, + } + ) + @patch("onadata.apps.messaging.signals.call_backend") def test_messaging_backends_handler(self, call_backend_mock): """ Test messaging backends handler function. @@ -49,4 +50,5 @@ def test_messaging_backends_handler(self, call_backend_mock): messaging_backends_handler(Action, instance=Action(id=9), created=True) self.assertTrue(call_backend_mock.called) call_backend_mock.assert_called_with( - 'onadata.apps.messaging.backends.base.BaseBackend', 9, None) + "onadata.apps.messaging.backends.base.BaseBackend", 9, None + ) diff --git a/onadata/apps/restservice/tests/test_restservice.py b/onadata/apps/restservice/tests/test_restservice.py index 21ea036b34..d85f6d6657 100644 --- a/onadata/apps/restservice/tests/test_restservice.py +++ b/onadata/apps/restservice/tests/test_restservice.py @@ -4,12 +4,11 @@ """ import os import time +from unittest.mock import patch from django.test.utils import override_settings from django.urls import reverse -from mock import patch - from onadata.apps.logger.models.xform import XForm from onadata.apps.main.models import MetaData from onadata.apps.main.tests.test_base import TestBase diff --git a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py index 19ad06421a..af2052e068 100644 --- a/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py +++ b/onadata/apps/restservice/tests/viewsets/test_restservicesviewset.py @@ -2,9 +2,9 @@ """ Test /restservices API endpoint implementation. """ -from django.test.utils import override_settings +from unittest.mock import patch -from mock import patch +from django.test.utils import override_settings from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.main.models.meta_data import MetaData diff --git a/onadata/apps/viewer/tests/test_attachment_url.py b/onadata/apps/viewer/tests/test_attachment_url.py index 3a7a8f31c6..bf5b4c2a2b 100644 --- a/onadata/apps/viewer/tests/test_attachment_url.py +++ b/onadata/apps/viewer/tests/test_attachment_url.py @@ -1,10 +1,15 @@ +# -*- coding: utf-8 -*- +""" +Test attachments. +""" import os +from unittest.mock import patch from django.conf import settings from django.contrib.auth import authenticate from django.http import HttpResponseRedirect from django.urls import reverse -from mock import patch + from rest_framework.test import APIRequestFactory from onadata.apps.logger.models import Attachment @@ -14,6 +19,9 @@ class TestAttachmentUrl(TestBase): + """ + Test attachments. + """ def setUp(self): self.attachment_count = 0 @@ -21,51 +29,49 @@ def setUp(self): self._create_user_and_login() self._publish_transportation_form() self._submit_transport_instance_w_attachment() - self.url = reverse( - attachment_url, kwargs={'size': 'original'}) + self.url = reverse(attachment_url, kwargs={"size": "original"}) self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) def test_attachment_url(self): - self.assertEqual( - Attachment.objects.count(), self.attachment_count + 1) + self.assertEqual(Attachment.objects.count(), self.attachment_count + 1) response = self.client.get( - self.url, {"media_file": self.attachment_media_file.name}) + self.url, {"media_file": self.attachment_media_file.name} + ) self.assertEqual(response.status_code, 302) # redirects to amazon def test_attachment_url_no_redirect(self): - self.assertEqual( - Attachment.objects.count(), self.attachment_count + 1) + self.assertEqual(Attachment.objects.count(), self.attachment_count + 1) response = self.client.get( - self.url, {"media_file": self.attachment_media_file.name, - 'no_redirect': 'true'}) + self.url, + {"media_file": self.attachment_media_file.name, "no_redirect": "true"}, + ) self.assertEqual(response.status_code, 200) # no redirects to amazon def test_attachment_not_found(self): response = self.client.get( - self.url, {"media_file": "non_existent_attachment.jpg"}) + self.url, {"media_file": "non_existent_attachment.jpg"} + ) self.assertEqual(response.status_code, 404) def test_attachment_has_mimetype(self): attachment = Attachment.objects.all().reverse()[0] - self.assertEqual(attachment.mimetype, 'image/jpeg') + self.assertEqual(attachment.mimetype, "image/jpeg") def test_attachment_url_w_media_id(self): """Test attachment url with attachment id""" - self.assertEqual( - Attachment.objects.count(), self.attachment_count + 1) - response = self.client.get( - self.url, {"attachment_id": self.attachment.id}) + self.assertEqual(Attachment.objects.count(), self.attachment_count + 1) + response = self.client.get(self.url, {"attachment_id": self.attachment.id}) self.assertEqual(response.status_code, 302) # redirects to amazon # pylint: disable=invalid-name def test_attachment_url_w_media_id_no_redirect(self): """Test attachment url with attachment id no redirect""" - self.assertEqual( - Attachment.objects.count(), self.attachment_count + 1) + self.assertEqual(Attachment.objects.count(), self.attachment_count + 1) response = self.client.get( - self.url, {"attachment_id": self.attachment.id, - 'no_redirect': 'true'}) + self.url, {"attachment_id": self.attachment.id, "no_redirect": "true"} + ) self.assertEqual(response.status_code, 200) # no redirects to amazon @patch("onadata.apps.viewer.views.generate_media_download_url") @@ -108,7 +114,7 @@ def test_attachment_url_has_azure_sas_token(self, mock_media_url): # get submission enc attachment attachment = Attachment.objects.all()[1] - sas_token = "se=ab736fba7261" # nosec + sas_token = "se=ab736fba7261" # nosec expected_url = f"http://testserver/{attachment.media_file.name}?{sas_token}" mock_media_url.return_value = HttpResponseRedirect(redirect_to=expected_url) response = self.client.get(self.url, {"media_file": attachment.media_file.name}) diff --git a/onadata/apps/viewer/tests/test_exports.py b/onadata/apps/viewer/tests/test_exports.py index ce5393c463..55b8264523 100644 --- a/onadata/apps/viewer/tests/test_exports.py +++ b/onadata/apps/viewer/tests/test_exports.py @@ -1,49 +1,53 @@ +# -*- coding: utf-8 -*- +""" +Test exports +""" import csv import datetime import json import os from io import StringIO from time import sleep +from unittest.mock import patch -import openpyxl - -from celery import current_app from django.conf import settings from django.core.files.storage import get_storage_class from django.http import Http404 from django.urls import reverse from django.utils.dateparse import parse_datetime -from mock import patch + +import openpyxl +from celery import current_app from onadata.apps.logger.models import Instance from onadata.apps.main.models.meta_data import MetaData from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.views import delete_data from onadata.apps.viewer.models.export import Export -from onadata.apps.viewer.models.parsed_instance import query_data, query_count +from onadata.apps.viewer.models.parsed_instance import query_count, query_data from onadata.apps.viewer.tasks import create_xlsx_export from onadata.apps.viewer.tests.export_helpers import viewer_fixture_path from onadata.apps.viewer.views import ( + create_export, delete_export, + export_download, export_list, - create_export, export_progress, - export_download, ) from onadata.apps.viewer.xls_writer import XlsWriter from onadata.libs.utils.common_tools import get_response_content from onadata.libs.utils.export_builder import dict_to_joined_export from onadata.libs.utils.export_tools import ( + clean_keys_of_slashes, generate_export, increment_index_in_filename, - clean_keys_of_slashes, ) AMBULANCE_KEY = ( - "transport/available_transportation_types_to_referral_fac" "ility/ambulance" + "transport/available_transportation_types_to_referral_facility/ambulance" ) AMBULANCE_KEY_DOTS = ( - "transport.available_transportation_types_to_referra" "l_facility.ambulance" + "transport.available_transportation_types_to_referral_facility.ambulance" ) diff --git a/onadata/libs/tests/data/test_statistics.py b/onadata/libs/tests/data/test_statistics.py index b3c4844055..f65735dcb5 100644 --- a/onadata/libs/tests/data/test_statistics.py +++ b/onadata/libs/tests/data/test_statistics.py @@ -1,9 +1,17 @@ +# -*- coding: utf-8 -*- +""" +Test onadata.libs.data module +""" import unittest from onadata.libs.data import statistics as stats class TestStatistics(unittest.TestCase): + """ + Test onadata.libs.data module + """ + def test_get_mean(self): values = [1, 2, 3, 2, 5, 5] result = stats.get_mean(values) diff --git a/onadata/libs/tests/data/test_tools.py b/onadata/libs/tests/data/test_tools.py index 652d7e8523..cf81c8cee9 100644 --- a/onadata/libs/tests/data/test_tools.py +++ b/onadata/libs/tests/data/test_tools.py @@ -1,19 +1,27 @@ -from datetime import datetime, timedelta -from django.utils.timezone import utc +# -*- coding: utf-8 -*- +""" +Test onadata.libs.data.query module +""" import os +from datetime import datetime, timedelta +from unittest.mock import patch -from mock import patch +from django.utils.timezone import utc from onadata.apps.logger.models.instance import Instance from onadata.apps.main.tests.test_base import TestBase from onadata.libs.data.query import ( - get_form_submissions_grouped_by_field, get_date_fields, get_field_records, + get_form_submissions_grouped_by_field, ) class TestTools(TestBase): + """ + Test onadata.libs.data.query module + """ + def setUp(self): super().setUp() self._create_user_and_login() diff --git a/onadata/libs/tests/serializers/test_project_serializer.py b/onadata/libs/tests/serializers/test_project_serializer.py index 2775eec4bb..7a0b86774b 100644 --- a/onadata/libs/tests/serializers/test_project_serializer.py +++ b/onadata/libs/tests/serializers/test_project_serializer.py @@ -1,17 +1,28 @@ -from mock import MagicMock +# -*- coding: utf-8 -*- +""" +Test onadata.libs.serializers.project_serializer +""" +from unittest.mock import MagicMock + +from django.core.cache import cache + from rest_framework import serializers from rest_framework.test import APIRequestFactory -from django.core.cache import cache -from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key -from onadata.apps.api.tests.viewsets.test_abstract_viewset import \ - TestAbstractViewSet +from onadata.apps.api.tests.viewsets.test_abstract_viewset import TestAbstractViewSet from onadata.apps.logger.models import Project -from onadata.libs.serializers.project_serializer import (BaseProjectSerializer, - ProjectSerializer) +from onadata.libs.serializers.project_serializer import ( + BaseProjectSerializer, + ProjectSerializer, +) +from onadata.libs.utils.cache_tools import PROJ_OWNER_CACHE, safe_key class TestBaseProjectSerializer(TestAbstractViewSet): + """ + Test onadata.libs.serializers.project_serializer + """ + def setUp(self): self.factory = APIRequestFactory() self._login_user_and_profile() @@ -19,46 +30,54 @@ def setUp(self): self._org_create() data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' - % self.organization.user.username, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False - } + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" + % self.organization.user.username, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } # Create the project self._project_create(data) def test_get_users(self): - "" + """""" # Is none when request to get users lacks a project users = self.serializer.get_users(None) self.assertEqual(users, None) # Has members and NOT collaborators when NOT passed 'owner' - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user - self.serializer.context['request'] = request + self.serializer.context["request"] = request users = self.serializer.get_users(self.project) - self.assertEqual(sorted(users, key=lambda x: x['first_name']), - [{'first_name': u'Bob', - 'last_name': u'erama', - 'is_org': False, - 'role': 'owner', - 'user': u'bob', - 'metadata': {}}, - {'first_name': u'Dennis', - 'last_name': u'', - 'is_org': True, - 'role': 'owner', - 'user': u'denoinc', - 'metadata': {}}]) + self.assertEqual( + sorted(users, key=lambda x: x["first_name"]), + [ + { + "first_name": "Bob", + "last_name": "erama", + "is_org": False, + "role": "owner", + "user": "bob", + "metadata": {}, + }, + { + "first_name": "Dennis", + "last_name": "", + "is_org": True, + "role": "owner", + "user": "denoinc", + "metadata": {}, + }, + ], + ) class TestProjectSerializer(TestAbstractViewSet): - def setUp(self): self.serializer = ProjectSerializer() self.factory = APIRequestFactory() @@ -75,40 +94,42 @@ def test_get_project_forms(self): project = Project.objects.last() form = project.xform_set.last() - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user serializer = ProjectSerializer(project) - serializer.context['request'] = request + serializer.context["request"] = request - self.assertEqual(len(serializer.data['forms']), 1) - self.assertEqual(serializer.data['forms'][0]['encrypted'], False) - self.assertEqual(serializer.data['num_datasets'], 1) + self.assertEqual(len(serializer.data["forms"]), 1) + self.assertEqual(serializer.data["forms"][0]["encrypted"], False) + self.assertEqual(serializer.data["num_datasets"], 1) # delete form in project form.delete() # Check that project has no forms self.assertIsNone(project.xform_set.last()) - serializer = ProjectSerializer(project, context={'request': request}) - self.assertEqual(len(serializer.data['forms']), 0) - self.assertEqual(serializer.data['num_datasets'], 0) + serializer = ProjectSerializer(project, context={"request": request}) + self.assertEqual(len(serializer.data["forms"]), 0) + self.assertEqual(serializer.data["num_datasets"], 0) def test_create_duplicate_projects(self): validated_data = { - 'name': u'demo', - 'organization': self.user, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False - } + "name": "demo", + "organization": self.user, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } # create first project request = MagicMock(user=self.user) - serializer = ProjectSerializer(context={'request': request}) + serializer = ProjectSerializer(context={"request": request}) project = serializer.create(validated_data) - self.assertEqual(project.name, u'demo') + self.assertEqual(project.name, "demo") self.assertEqual(project.organization, self.user) # create another project with same data @@ -116,37 +137,36 @@ def test_create_duplicate_projects(self): serializer.create(validated_data) self.assertEqual( e.exception.detail, - [u'The fields name, organization must make a unique set.']) + ["The fields name, organization must make a unique set."], + ) def test_new_project_set_to_cache(self): """ Test that newly created project is set to cache """ data = { - 'name': u'demo', - 'owner': - 'http://testserver/api/v1/users/%s' - % self.user, - 'metadata': {'description': 'Some description', - 'location': 'Naivasha, Kenya', - 'category': 'governance'}, - 'public': False - } + "name": "demo", + "owner": "http://testserver/api/v1/users/%s" % self.user, + "metadata": { + "description": "Some description", + "location": "Naivasha, Kenya", + "category": "governance", + }, + "public": False, + } # clear cache - cache.delete(safe_key(f'{PROJ_OWNER_CACHE}1')) - self.assertIsNone(cache.get(safe_key(f'{PROJ_OWNER_CACHE}1'))) + cache.delete(safe_key(f"{PROJ_OWNER_CACHE}1")) + self.assertIsNone(cache.get(safe_key(f"{PROJ_OWNER_CACHE}1"))) # Create the project self._project_create(data) self.assertIsNotNone(self.project_data) - request = self.factory.get('/', **self.extra) + request = self.factory.get("/", **self.extra) request.user = self.user - serializer = ProjectSerializer( - self.project, context={'request': request}).data - self.assertEqual( - cache.get(f'{PROJ_OWNER_CACHE}{self.project.pk}'), serializer) + serializer = ProjectSerializer(self.project, context={"request": request}).data + self.assertEqual(cache.get(f"{PROJ_OWNER_CACHE}{self.project.pk}"), serializer) # clear cache - cache.delete(safe_key(f'{PROJ_OWNER_CACHE}{self.project.pk}')) + cache.delete(safe_key(f"{PROJ_OWNER_CACHE}{self.project.pk}")) diff --git a/onadata/libs/tests/serializers/test_xform_serializer.py b/onadata/libs/tests/serializers/test_xform_serializer.py index d6ba035198..36d8b9f90f 100644 --- a/onadata/libs/tests/serializers/test_xform_serializer.py +++ b/onadata/libs/tests/serializers/test_xform_serializer.py @@ -2,8 +2,9 @@ """ Test onadata.libs.serializers.xform_serializer """ +from unittest.mock import MagicMock + from django.test import TestCase -from mock import MagicMock from onadata.apps.main.tests.test_base import TestBase from onadata.libs.serializers.xform_serializer import XFormManifestSerializer @@ -22,10 +23,10 @@ def test_get_filename_from_url(self): serializer = XFormManifestSerializer() obj.data_value = "http://example.com/" - self.assertEqual(serializer.get_filename(obj), 'example.com') + self.assertEqual(serializer.get_filename(obj), "example.com") obj.data_value = "http://example.com/clinics.csv" - self.assertEqual(serializer.get_filename(obj), 'clinics.csv') + self.assertEqual(serializer.get_filename(obj), "clinics.csv") # pylint: disable=C0103 def test_get_filename_form_filtered_dataset(self): @@ -36,7 +37,7 @@ def test_get_filename_form_filtered_dataset(self): serializer = XFormManifestSerializer() obj.data_value = "xform 1 clinics" - self.assertEqual(serializer.get_filename(obj), 'clinics.csv') + self.assertEqual(serializer.get_filename(obj), "clinics.csv") def test_get_hash(self): """ @@ -54,7 +55,7 @@ def test_get_hash(self): obj.data_value = "xform {} test_dataset".format(self.xform.id) - obj.file_hash = u'md5:b9cc8695c526f3c7aaa882234f3b9484' + obj.file_hash = "md5:b9cc8695c526f3c7aaa882234f3b9484" obj.data_file = "" self.assertNotEqual(serializer.get_hash(obj), obj.file_hash) diff --git a/onadata/libs/tests/test_authentication.py b/onadata/libs/tests/test_authentication.py index caba609e8d..e19f4ab66d 100644 --- a/onadata/libs/tests/test_authentication.py +++ b/onadata/libs/tests/test_authentication.py @@ -1,27 +1,26 @@ -import jwt from datetime import timedelta +from unittest.mock import MagicMock, patch from django.conf import settings from django.contrib.auth.models import User -from django.test import TestCase from django.http.request import HttpRequest +from django.test import TestCase -from mock import patch, MagicMock +import jwt from oauth2_provider.models import AccessToken from rest_framework.exceptions import AuthenticationFailed from rest_framework.test import APIRequestFactory from onadata.apps.api.models.temp_token import TempToken -from onadata.libs.utils.common_tags import API_TOKEN from onadata.libs.authentication import ( DigestAuthentication, + MasterReplicaOAuth2Validator, TempTokenAuthentication, TempTokenURLParameterAuthentication, check_lockout, get_api_token, - MasterReplicaOAuth2Validator, ) - +from onadata.libs.utils.common_tags import API_TOKEN JWT_SECRET_KEY = getattr(settings, "JWT_SECRET_KEY", "jwt") JWT_ALGORITHM = getattr(settings, "JWT_ALGORITHM", "HS256") diff --git a/onadata/libs/tests/test_permissions.py b/onadata/libs/tests/test_permissions.py index 00e65a956e..a9ad1b218d 100644 --- a/onadata/libs/tests/test_permissions.py +++ b/onadata/libs/tests/test_permissions.py @@ -2,17 +2,27 @@ """ Tests onadata.libs.permissions module """ +from unittest.mock import patch + from django.contrib.auth.models import Group + from guardian.shortcuts import get_users_with_perms -from mock import patch from onadata.apps.api import tools from onadata.apps.main.models.user_profile import UserProfile from onadata.apps.main.tests.test_base import TestBase from onadata.libs.permissions import ( - CAN_ADD_XFORM_TO_PROFILE, DataEntryMinorRole, EditorRole, ManagerRole, - NoRecordsPermission, OwnerRole, ReadOnlyRole, ReadOnlyRoleNoDownload, - filter_queryset_xform_meta_perms_sql, get_object_users_with_permissions) + CAN_ADD_XFORM_TO_PROFILE, + DataEntryMinorRole, + EditorRole, + ManagerRole, + NoRecordsPermission, + OwnerRole, + ReadOnlyRole, + ReadOnlyRoleNoDownload, + filter_queryset_xform_meta_perms_sql, + get_object_users_with_permissions, +) def perms_for(user, obj): @@ -26,12 +36,13 @@ class TestPermissions(TestBase): """ Tests for onadata.libs.permissions module """ + def test_manager_role_add(self): """ Test adding ManagerRole """ bob, _ = UserProfile.objects.get_or_create(user=self.user) - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") self.assertFalse(alice.has_perm(CAN_ADD_XFORM_TO_PROFILE, bob)) ManagerRole.add(alice, bob) @@ -43,7 +54,7 @@ def test_manager_has_role(self): Test manager has role """ bob, _ = UserProfile.objects.get_or_create(user=self.user) - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") self.assertFalse(ManagerRole.user_has_role(alice, bob)) self.assertFalse(ManagerRole.has_role(perms_for(alice, bob), bob)) @@ -58,24 +69,21 @@ def test_reassign_role(self): Test role reassignment. """ self._publish_transportation_form() - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") self.assertFalse(ManagerRole.user_has_role(alice, self.xform)) ManagerRole.add(alice, self.xform) self.assertTrue(ManagerRole.user_has_role(alice, self.xform)) - self.assertTrue( - ManagerRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertTrue(ManagerRole.has_role(perms_for(alice, self.xform), self.xform)) ReadOnlyRole.add(alice, self.xform) self.assertFalse(ManagerRole.user_has_role(alice, self.xform)) self.assertTrue(ReadOnlyRole.user_has_role(alice, self.xform)) - self.assertFalse( - ManagerRole.has_role(perms_for(alice, self.xform), self.xform)) - self.assertTrue( - ReadOnlyRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertFalse(ManagerRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertTrue(ReadOnlyRole.has_role(perms_for(alice, self.xform), self.xform)) # pylint: disable=C0103 def test_reassign_role_owner_to_editor(self): @@ -83,83 +91,82 @@ def test_reassign_role_owner_to_editor(self): Test role reassignment owner to editor. """ self._publish_transportation_form() - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") self.assertFalse(OwnerRole.user_has_role(alice, self.xform)) OwnerRole.add(alice, self.xform) self.assertTrue(OwnerRole.user_has_role(alice, self.xform)) - self.assertTrue( - OwnerRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertTrue(OwnerRole.has_role(perms_for(alice, self.xform), self.xform)) EditorRole.add(alice, self.xform) self.assertFalse(OwnerRole.user_has_role(alice, self.xform)) self.assertTrue(EditorRole.user_has_role(alice, self.xform)) - self.assertFalse( - OwnerRole.has_role(perms_for(alice, self.xform), self.xform)) - self.assertTrue( - EditorRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertFalse(OwnerRole.has_role(perms_for(alice, self.xform), self.xform)) + self.assertTrue(EditorRole.has_role(perms_for(alice, self.xform), self.xform)) # pylint: disable=C0103 def test_get_object_users_with_permission(self): """ Test get_object_users_with_permissions() """ - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.get_or_create(user=alice) org_user = tools.create_organization("modilabs", alice).user - demo_grp = Group.objects.create(name='demo') + demo_grp = Group.objects.create(name="demo") alice.groups.add(demo_grp) self._publish_transportation_form() EditorRole.add(org_user, self.xform) EditorRole.add(demo_grp, self.xform) users_with_perms = get_object_users_with_permissions( - self.xform, with_group_users=True) - self.assertTrue(org_user in [d['user'] for d in users_with_perms]) - self.assertTrue(alice in [d['user'] for d in users_with_perms]) + self.xform, with_group_users=True + ) + self.assertTrue(org_user in [d["user"] for d in users_with_perms]) + self.assertTrue(alice in [d["user"] for d in users_with_perms]) users_with_perms_first_keys = list(users_with_perms[0]) - self.assertIn('first_name', users_with_perms_first_keys) - self.assertIn('last_name', users_with_perms_first_keys) - self.assertIn('user', users_with_perms_first_keys) - self.assertIn('role', users_with_perms_first_keys) - self.assertIn('gravatar', users_with_perms_first_keys) - self.assertIn('metadata', users_with_perms_first_keys) - self.assertIn('is_org', users_with_perms_first_keys) + self.assertIn("first_name", users_with_perms_first_keys) + self.assertIn("last_name", users_with_perms_first_keys) + self.assertIn("user", users_with_perms_first_keys) + self.assertIn("role", users_with_perms_first_keys) + self.assertIn("gravatar", users_with_perms_first_keys) + self.assertIn("metadata", users_with_perms_first_keys) + self.assertIn("is_org", users_with_perms_first_keys) # pylint: disable=C0103 def test_user_profile_exists_for_users_with_perms(self): """ Test user profile exists when retrieving users with perms """ - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # do not manually create user profile for alice, should be # handled by get_object_users_with_permissions() org_user = tools.create_organization("modilabs", alice).user - demo_grp = Group.objects.create(name='demo') + demo_grp = Group.objects.create(name="demo") alice.groups.add(demo_grp) self._publish_transportation_form() EditorRole.add(org_user, self.xform) EditorRole.add(demo_grp, self.xform) users_with_perms = get_object_users_with_permissions( - self.xform, with_group_users=True) - self.assertTrue(org_user in [d['user'] for d in users_with_perms]) - self.assertTrue(alice in [d['user'] for d in users_with_perms]) + self.xform, with_group_users=True + ) + self.assertTrue(org_user in [d["user"] for d in users_with_perms]) + self.assertTrue(alice in [d["user"] for d in users_with_perms]) for d in users_with_perms: - user_obj = d['user'] + user_obj = d["user"] self.assertTrue(hasattr(user_obj, "profile")) - # pylint: disable=C0103 + # pylint: disable=C0103 def test_exception_raised_for_missing_profiles(self): """ Test UserProfile.DoesNotExit exception raised for missing user profiles """ - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") UserProfile.objects.get_or_create(user=alice) org_user = tools.create_organization("modilabs", alice).user - demo_grp = Group.objects.create(name='demo') + demo_grp = Group.objects.create(name="demo") alice.groups.add(demo_grp) self._publish_transportation_form() EditorRole.add(org_user, self.xform) @@ -177,32 +184,31 @@ def test_exception_raised_for_missing_profiles(self): # check if profile is created for alice # when get_object_users_with_permissions() is called users_with_perms = get_object_users_with_permissions( - self.xform, with_group_users=True) - self.assertEqual("alice", users_with_perms[2]['user'].username) - self.assertTrue(hasattr(users_with_perms[2]['user'], "profile")) + self.xform, with_group_users=True + ) + self.assertEqual("alice", users_with_perms[2]["user"].username) + self.assertTrue(hasattr(users_with_perms[2]["user"], "profile")) def test_readonly_no_downloads_has_role(self): """ Test readonly no downloads role. """ self._publish_transportation_form() - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") + self.assertFalse(ReadOnlyRoleNoDownload.user_has_role(alice, self.xform)) self.assertFalse( - ReadOnlyRoleNoDownload.user_has_role(alice, self.xform)) - self.assertFalse( - ReadOnlyRoleNoDownload.has_role( - perms_for(alice, self.xform), self.xform)) + ReadOnlyRoleNoDownload.has_role(perms_for(alice, self.xform), self.xform) + ) ReadOnlyRoleNoDownload.add(alice, self.xform) + self.assertTrue(ReadOnlyRoleNoDownload.user_has_role(alice, self.xform)) self.assertTrue( - ReadOnlyRoleNoDownload.user_has_role(alice, self.xform)) - self.assertTrue( - ReadOnlyRoleNoDownload.has_role( - perms_for(alice, self.xform), self.xform)) + ReadOnlyRoleNoDownload.has_role(perms_for(alice, self.xform), self.xform) + ) - @patch('onadata.libs.permissions._check_meta_perms_enabled') + @patch("onadata.libs.permissions._check_meta_perms_enabled") def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock): """ Test filter query by meta permissions. @@ -210,12 +216,11 @@ def test_filter_queryset_xform_meta_perms_sql(self, check_meta_mock): self._publish_transportation_form() query = '{"_id": 1}' - result = filter_queryset_xform_meta_perms_sql(self.xform, self.user, - query) + result = filter_queryset_xform_meta_perms_sql(self.xform, self.user, query) self.assertEqual(result, query) check_meta_mock.return_value = True - alice = self._create_user('alice', 'alice') + alice = self._create_user("alice", "alice") # no records with self.assertRaises(NoRecordsPermission): diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index fb55f8ce39..13a31d11c0 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -4,27 +4,26 @@ import re from builtins import open from io import BytesIO +from unittest.mock import patch from xml.etree.ElementTree import fromstring -import mock +from django.conf import settings + import unicodecsv as ucsv from celery.backends.rpc import BacklogLimitExceeded -from django.conf import settings -from mock import patch from onadata.apps.logger.models import Instance, XForm -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.models import MetaData +from onadata.apps.main.tests.test_base import TestBase from onadata.apps.messaging.constants import ( - XFORM, - SUBMISSION_EDITED, SUBMISSION_CREATED, + SUBMISSION_EDITED, + XFORM, ) from onadata.libs.utils import csv_import from onadata.libs.utils.common_tags import IMPORTED_VIA_CSV_BY -from onadata.libs.utils.csv_import import get_submission_meta_dict +from onadata.libs.utils.csv_import import get_columns_by_type, get_submission_meta_dict from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.csv_import import get_columns_by_type def strip_xml_uuid(s): @@ -60,7 +59,7 @@ def test_submit_csv_param_sanity_check(self): resp = csv_import.submit_csv("userX", XForm(), 123456) self.assertIsNotNone(resp.get("error")) - @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") + @patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_xml_params(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() @@ -87,8 +86,8 @@ def test_submit_csv_xml_params(self, safe_create_instance): ) self.assertEqual(safe_create_args[4], None) - @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") - @mock.patch("onadata.libs.utils.csv_import.dict2xmlsubmission") + @patch("onadata.libs.utils.csv_import.safe_create_instance") + @patch("onadata.libs.utils.csv_import.dict2xmlsubmission") def test_submit_csv_xml_location_property_test(self, d2x, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() @@ -173,7 +172,7 @@ def test_submit_csv_edits(self, send_message_mock): ) # message sent upon submission edit self.assertTrue(send_message_mock.called) - send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_EDITED) + send_message_mock.assert_called_with(self.xform.id, XFORM, SUBMISSION_EDITED) def test_import_non_utf8_csv(self): xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xlsx") @@ -309,7 +308,7 @@ def test_csv_with__more_than_4_repeats_import(self): # repeats should be 6 self.assertEqual(6, len(instance.json.get("children"))) - @mock.patch("onadata.libs.utils.csv_import.AsyncResult") + @patch("onadata.libs.utils.csv_import.AsyncResult") def test_get_async_csv_submission_status(self, AsyncResult): result = csv_import.get_async_csv_submission_status(None) self.assertEqual(result, {"error": "Empty job uuid", "job_status": "FAILURE"}) @@ -383,7 +382,7 @@ def test_submission_xls_to_csv(self): self.assertEqual(g_csv_reader.fieldnames[10], c_csv_reader.fieldnames[10]) - @mock.patch("onadata.libs.utils.csv_import.safe_create_instance") + @patch("onadata.libs.utils.csv_import.safe_create_instance") def test_submit_csv_instance_id_consistency(self, safe_create_instance): self._publish_xls_file(self.xls_file_path) self.xform = XForm.objects.get() @@ -426,7 +425,7 @@ def test_data_upload(self, send_message_mock): self.assertEqual(self.xform.num_of_submissions, count + 1) # message sent upon submission creation self.assertTrue(send_message_mock.called) - send_message_mock.called_with(self.xform.id, XFORM, SUBMISSION_CREATED) + send_message_mock.assert_called_with(self.xform.id, XFORM, SUBMISSION_CREATED) def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" diff --git a/onadata/libs/tests/utils/test_email.py b/onadata/libs/tests/utils/test_email.py index b1732b515b..bcff7b6f7a 100644 --- a/onadata/libs/tests/utils/test_email.py +++ b/onadata/libs/tests/utils/test_email.py @@ -1,22 +1,30 @@ -from six.moves.urllib.parse import urlencode -from mock import patch +# -*- coding: utf-8 -*- +""" +Test onadata.utils.emails module. +""" +from unittest.mock import patch + from django.test import RequestFactory from django.test.utils import override_settings + +from six.moves.urllib.parse import urlencode + +from onadata.apps.logger.models import ProjectInvitation from onadata.apps.main.tests.test_base import TestBase from onadata.libs.utils.email import ( + ProjectInvitationEmail, + get_project_invitation_url, get_verification_email_data, get_verification_url, - get_project_invitation_url, ) -from onadata.libs.utils.email import ProjectInvitationEmail -from onadata.apps.logger.models import ProjectInvitation from onadata.libs.utils.user_auth import get_user_default_project - VERIFICATION_URL = "http://ab.cd.ef" class TestEmail(TestBase): + """Test onadata.utils.email module""" + def setUp(self): self.email = "john@doe.com" self.username = ("johndoe",) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index c158b62268..e4aeee65ef 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -13,10 +13,10 @@ from collections import OrderedDict from ctypes import ArgumentError from io import BytesIO +from unittest.mock import patch from django.conf import settings from django.core.files.temp import NamedTemporaryFile -from mock import patch from openpyxl import load_workbook from pyxform.builder import create_survey_from_xls diff --git a/onadata/libs/tests/utils/test_logger_tools.py b/onadata/libs/tests/utils/test_logger_tools.py index 03516b7e66..ff0a74fd2b 100644 --- a/onadata/libs/tests/utils/test_logger_tools.py +++ b/onadata/libs/tests/utils/test_logger_tools.py @@ -5,13 +5,13 @@ import os import re from io import BytesIO +from unittest.mock import patch from django.conf import settings from django.core.files.uploadedfile import InMemoryUploadedFile from django.http.request import HttpRequest from defusedxml.ElementTree import ParseError -from mock import patch from onadata.apps.logger.import_tools import django_file from onadata.apps.logger.models import Instance diff --git a/onadata/libs/tests/utils/test_project_utils.py b/onadata/libs/tests/utils/test_project_utils.py index a1267779ad..84696778c9 100644 --- a/onadata/libs/tests/utils/test_project_utils.py +++ b/onadata/libs/tests/utils/test_project_utils.py @@ -2,10 +2,11 @@ """ Test onadata.libs.utils.project_utils """ +from unittest.mock import MagicMock, patch + from django.test.utils import override_settings from kombu.exceptions import OperationalError -from mock import MagicMock, patch from requests import Response from onadata.apps.logger.models import Project diff --git a/onadata/libs/tests/utils/test_viewer_tools.py b/onadata/libs/tests/utils/test_viewer_tools.py index 2ed76c193f..3b0030b135 100644 --- a/onadata/libs/tests/utils/test_viewer_tools.py +++ b/onadata/libs/tests/utils/test_viewer_tools.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """Test onadata.libs.utils.viewer_tools.""" import os +from unittest.mock import patch from django.core.files.base import File from django.core.files.temp import NamedTemporaryFile @@ -9,8 +10,6 @@ from django.test.utils import override_settings from django.utils import timezone -from mock import patch - from onadata.apps.logger.models import Attachment, Instance, XForm from onadata.apps.main.tests.test_base import TestBase from onadata.libs.utils.viewer_tools import ( diff --git a/requirements/base.pip b/requirements/base.pip index 81b9beb2cc..d0ae022f23 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -41,9 +41,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.68 +boto3==1.34.69 # via dataflows-tabulator -botocore==1.34.68 +botocore==1.34.69 # via # boto3 # s3transfer @@ -74,7 +74,7 @@ click==8.1.7 # dataflows-tabulator # datapackage # tableschema -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery @@ -85,7 +85,7 @@ cryptography==42.0.5 # jwcrypto # onadata # pyjwt -dataflows-tabulator==1.54.1 +dataflows-tabulator==1.54.3 # via # datapackage # tableschema @@ -330,7 +330,7 @@ requests==2.31.0 # requests-oauthlib # sphinx # tableschema -requests-oauthlib==1.4.0 +requests-oauthlib==2.0.0 # via google-auth-oauthlib rfc3986==2.0.0 # via tableschema @@ -372,7 +372,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy==2.0.28 +sqlalchemy==2.0.29 # via dataflows-tabulator sqlparse==0.4.4 # via diff --git a/requirements/dev.in b/requirements/dev.in index 3762559028..7c5947dad6 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -6,7 +6,6 @@ flaky httmock ipdb isort -mock pre-commit prospector>=1.10.3 pylint diff --git a/requirements/dev.pip b/requirements/dev.pip index 89902aa040..8f3db50884 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -49,9 +49,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.68 +boto3==1.34.69 # via dataflows-tabulator -botocore==1.34.68 +botocore==1.34.69 # via # boto3 # s3transfer @@ -84,7 +84,7 @@ click==8.1.7 # dataflows-tabulator # datapackage # tableschema -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery @@ -95,7 +95,7 @@ cryptography==42.0.5 # jwcrypto # onadata # pyjwt -dataflows-tabulator==1.54.1 +dataflows-tabulator==1.54.3 # via # datapackage # tableschema @@ -348,7 +348,7 @@ platformdirs==4.2.0 # pylint # virtualenv # yapf -pre-commit==3.6.2 +pre-commit==3.7.0 # via -r requirements/dev.in prompt-toolkit==3.0.43 # via @@ -464,7 +464,7 @@ requests==2.31.0 # tableschema requests-mock==1.11.0 # via -r requirements/dev.in -requests-oauthlib==1.4.0 +requests-oauthlib==2.0.0 # via google-auth-oauthlib requirements-detector==1.2.2 # via prospector @@ -518,7 +518,7 @@ sphinxcontrib-qthelp==1.0.7 # via sphinx sphinxcontrib-serializinghtml==1.1.10 # via sphinx -sqlalchemy==2.0.28 +sqlalchemy==2.0.29 # via dataflows-tabulator sqlparse==0.4.4 # via diff --git a/setup.cfg b/setup.cfg index cea9d0d529..2ccf9ad228 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,7 +24,6 @@ tests_require = flake8 flaky httmock - mock requests-mock install_requires = Django==4.0,<5 From 013773699c2497048dc8b990e89331b81721db75 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 12:56:52 +0300 Subject: [PATCH 21/37] _submission_time is already in UTC so set to UTC --- onadata/libs/renderers/renderers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/libs/renderers/renderers.py b/onadata/libs/renderers/renderers.py index 15d5e49756..08f003b980 100644 --- a/onadata/libs/renderers/renderers.py +++ b/onadata/libs/renderers/renderers.py @@ -53,7 +53,7 @@ def floip_rows_list(data): """ try: _submission_time = ( - parse_datetime(data["_submission_time"]).astimezone(timezone.utc) + parse_datetime(data["_submission_time"]).replace(tzinfo=timezone.utc) ).isoformat() except ValueError: From 831fe07baa8ac351546050ba2b4487b36e3819cf Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 15:13:34 +0300 Subject: [PATCH 22/37] csv_import: Update assert_called_with tests --- onadata/libs/tests/utils/test_csv_import.py | 40 ++++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/onadata/libs/tests/utils/test_csv_import.py b/onadata/libs/tests/utils/test_csv_import.py index 13a31d11c0..461b46bafa 100644 --- a/onadata/libs/tests/utils/test_csv_import.py +++ b/onadata/libs/tests/utils/test_csv_import.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Tests the onadata.libs.utils.csv_import module +""" from __future__ import unicode_literals import os @@ -31,6 +35,10 @@ def strip_xml_uuid(s): class CSVImportTestCase(TestBase): + """ + Tests the onadata.libs.utils.csv_import module + """ + def setUp(self): super(CSVImportTestCase, self).setUp() self.fixtures_dir = os.path.join( @@ -149,9 +157,9 @@ def test_submit_csv_edits(self, send_message_mock): settings.PROJECT_ROOT, "apps", "main", "tests", "fixtures", "tutorial.xlsx" ) self._publish_xls_file(xls_file_path) - self.xform = XForm.objects.get() + xform = XForm.objects.get() - csv_import.submit_csv(self.user.username, self.xform, self.good_csv) + csv_import.submit_csv(self.user.username, xform, self.good_csv) self.assertEqual( Instance.objects.count(), 9, "submit_csv edits #1 test Failed!" ) @@ -166,13 +174,20 @@ def test_submit_csv_edits(self, send_message_mock): ) count = Instance.objects.count() - csv_import.submit_csv(self.user.username, self.xform, edit_csv) + csv_import.submit_csv(self.user.username, xform, edit_csv) self.assertEqual( Instance.objects.count(), count, "submit_csv edits #2 test Failed!" ) # message sent upon submission edit self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with(self.xform.id, XFORM, SUBMISSION_EDITED) + instance_id = xform.instances.filter().order_by("date_modified").last().pk + send_message_mock.assert_called_with( + instance_id=instance_id, + target_id=xform.id, + target_type=XFORM, + message_verb=SUBMISSION_EDITED, + user=self.user, + ) def test_import_non_utf8_csv(self): xls_file_path = os.path.join(self.fixtures_dir, "mali_health.xlsx") @@ -415,17 +430,24 @@ def test_submit_csv_instance_id_consistency(self, safe_create_instance): def test_data_upload(self, send_message_mock): """Data upload for submissions with no uuids""" self._publish_xls_file(self.xls_file_path) - self.xform = XForm.objects.get() + xform = XForm.objects.get() count = Instance.objects.count() single_csv = open( os.path.join(self.fixtures_dir, "single_data_upload.csv"), "rb" ) - csv_import.submit_csv(self.user.username, self.xform, single_csv) - self.xform.refresh_from_db() - self.assertEqual(self.xform.num_of_submissions, count + 1) + csv_import.submit_csv(self.user.username, xform, single_csv) + xform.refresh_from_db() + self.assertEqual(xform.num_of_submissions, count + 1) + instance_id = xform.instances.last().pk # message sent upon submission creation self.assertTrue(send_message_mock.called) - send_message_mock.assert_called_with(self.xform.id, XFORM, SUBMISSION_CREATED) + send_message_mock.assert_called_with( + instance_id=instance_id, + target_id=xform.id, + target_type=XFORM, + message_verb=SUBMISSION_CREATED, + user=self.user, + ) def test_excel_date_conversion(self): """Convert date from 01/01/1900 to 01-01-1900""" From ee94bb23703b9d442627e5951192680ea087090b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 17:38:40 +0300 Subject: [PATCH 23/37] start and end time filters are in UTC --- onadata/apps/main/tests/test_form_exports.py | 190 ++++++++++++------- onadata/apps/viewer/views.py | 6 +- 2 files changed, 123 insertions(+), 73 deletions(-) diff --git a/onadata/apps/main/tests/test_form_exports.py b/onadata/apps/main/tests/test_form_exports.py index 254aba0ea6..a9aa0f943a 100644 --- a/onadata/apps/main/tests/test_form_exports.py +++ b/onadata/apps/main/tests/test_form_exports.py @@ -1,3 +1,7 @@ +# -*- coding: utf-8 -*- +""" +Tests exports +""" import csv import os import tempfile @@ -17,40 +21,46 @@ class TestFormExports(TestBase): + """ + Tests exports + """ def setUp(self): TestBase.setUp(self) self._create_user_and_login() self._publish_transportation_form_and_submit_instance() - self.csv_url = reverse('csv_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string}) - self.xls_url = reverse('xlsx_export', kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string}) + self.csv_url = reverse( + "csv_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + self.xls_url = reverse( + "xlsx_export", + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) def _num_rows(self, content, export_format): def xls_rows(f): wb = openpyxl.load_workbook(filename=BytesIO(f)) - current_sheet = wb.get_sheet_by_name('data') + current_sheet = wb.get_sheet_by_name("data") return len(list(current_sheet.rows)) def csv_rows(f): - with tempfile.TemporaryFile('w+') as tmp: - tmp.write(f.decode('utf-8')) + with tempfile.TemporaryFile("w+") as tmp: + tmp.write(f.decode("utf-8")) tmp.seek(0) - return len([line for line in csv.reader(tmp)]) + return len(list(line for line in csv.reader(tmp))) + num_rows_fn = { - 'xls': xls_rows, - 'xlsx': xls_rows, - 'csv': csv_rows, + "xls": xls_rows, + "xlsx": xls_rows, + "csv": csv_rows, } return num_rows_fn[export_format](content) def test_csv_raw_export_name(self): - response = self.client.get(self.csv_url + '?raw=1') + response = self.client.get(self.csv_url + "?raw=1") self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], 'attachment;') + self.assertEqual(response["Content-Disposition"], "attachment;") def _filter_export_test(self, url, export_format): """ @@ -59,82 +69,106 @@ def _filter_export_test(self, url, export_format): """ time.sleep(1) # 1 survey exists before this time - start_time = timezone.now().strftime('%y_%m_%d_%H_%M_%S') + start_time = ( + timezone.now().astimezone(timezone.utc).strftime("%y_%m_%d_%H_%M_%S") + ) time.sleep(1) s = self.surveys[1] self._make_submission( - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml')) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + ) time.sleep(1) # 2 surveys exist before this time - end_time = timezone.now().strftime('%y_%m_%d_%H_%M_%S') + end_time = timezone.now().astimezone(timezone.utc).strftime("%y_%m_%d_%H_%M_%S") time.sleep(1) # 3 surveys exist in total s = self.surveys[2] self._make_submission( - os.path.join(self.this_directory, 'fixtures', - 'transportation', 'instances', s, s + '.xml')) + os.path.join( + self.this_directory, + "fixtures", + "transportation", + "instances", + s, + s + ".xml", + ) + ) # test restricting to before end time - params = {'end': end_time} + params = {"end": end_time} response = self.client.get(url, params) self.assertEqual(response.status_code, 200) content = get_response_content(response, decode=False) self.assertEqual(self._num_rows(content, export_format), 3) + # test restricting to after start time, thus excluding the initial # submission - params = {'start': start_time} + params = {"start": start_time} response = self.client.get(url, params) self.assertEqual(response.status_code, 200) content = get_response_content(response, decode=False) self.assertEqual(self._num_rows(content, export_format), 3) + # test no time restriction response = self.client.get(url) - self.assertEqual(response.status_code, 200) content = get_response_content(response, decode=False) self.assertEqual(self._num_rows(content, export_format), 4) + # test restricting to between start time and end time - params = {'start': start_time, 'end': end_time} + params = {"start": start_time, "end": end_time} response = self.client.get(url, params) self.assertEqual(response.status_code, 200) content = get_response_content(response, decode=False) self.assertEqual(self._num_rows(content, export_format), 2) def test_filter_by_date_csv(self): - self._filter_export_test(self.csv_url, 'csv') + self._filter_export_test(self.csv_url, "csv") def test_filter_by_date_xls(self): - self._filter_export_test(self.xls_url, 'xlsx') + self._filter_export_test(self.xls_url, "xlsx") def test_restrict_csv_export_if_not_shared(self): response = self.anon.get(self.csv_url) self.assertEqual(response.status_code, 403) def test_xls_raw_export_name(self): - response = self.client.get(self.xls_url + '?raw=1') + response = self.client.get(self.xls_url + "?raw=1") self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], 'attachment;') + self.assertEqual(response["Content-Disposition"], "attachment;") def test_restrict_xlsx_export_if_not_shared(self): response = self.anon.get(self.xls_url) self.assertEqual(response.status_code, 403) def test_zip_raw_export_name(self): - url = reverse(zip_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) - response = self.client.get(url + '?raw=1') + url = reverse( + zip_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + response = self.client.get(url + "?raw=1") self.assertEqual(response.status_code, 200) - self.assertEqual(response['Content-Disposition'], 'attachment;') + self.assertEqual(response["Content-Disposition"], "attachment;") def test_restrict_zip_export_if_not_shared(self): - url = reverse(zip_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + zip_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url) self.assertEqual(response.status_code, 403) def test_restrict_kml_export_if_not_shared(self): - url = reverse(kml_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + kml_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url) self.assertEqual(response.status_code, 403) @@ -153,16 +187,20 @@ def test_allow_xlsx_export_if_shared(self): def test_allow_zip_export_if_shared(self): self.xform.shared_data = True self.xform.save() - url = reverse(zip_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + zip_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url) self.assertEqual(response.status_code, 200) def test_allow_kml_export_if_shared(self): self.xform.shared_data = True self.xform.save() - url = reverse(kml_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + kml_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url) self.assertEqual(response.status_code, 200) @@ -175,72 +213,84 @@ def test_allow_xlsx_export(self): self.assertEqual(response.status_code, 200) def test_allow_zip_export(self): - url = reverse(zip_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + zip_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_allow_kml_export(self): - url = reverse(kml_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + kml_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_allow_csv_export_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, - self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } response = self.anon.get(self.csv_url, **extra) self.assertEqual(response.status_code, 200) def test_allow_xlsx_export_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, - self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } response = self.anon.get(self.xls_url, **extra) self.assertEqual(response.status_code, 200) def test_allow_zip_export_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, - self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } - url = reverse(zip_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + zip_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url, **extra) self.assertEqual(response.status_code, 200) def test_allow_kml_export_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, - self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } - url = reverse(kml_export, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + url = reverse( + kml_export, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.anon.get(url, **extra) self.assertEqual(response.status_code, 200) def test_allow_export_download_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': http_auth_string(self.login_username, - self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } # create export options = {"extension": "csv"} - export = generate_export( - Export.CSV_EXPORT, - self.xform, - None, - options) + export = generate_export(Export.CSV_EXPORT, self.xform, None, options) self.assertTrue(isinstance(export, Export)) - url = reverse(export_download, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': export.export_type, - 'filename': export.filename - }) + url = reverse( + export_download, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": export.export_type, + "filename": export.filename, + }, + ) response = self.anon.get(url, **extra) self.assertEqual(response.status_code, 200) diff --git a/onadata/apps/viewer/views.py b/onadata/apps/viewer/views.py index 8b8c05a22f..41516d01b3 100644 --- a/onadata/apps/viewer/views.py +++ b/onadata/apps/viewer/views.py @@ -84,10 +84,10 @@ def _get_start_end_submission_time(request): if request.GET.get("start"): start = datetime.strptime( request.GET["start"], "%y_%m_%d_%H_%M_%S" - ).astimezone(timezone.utc) + ).replace(tzinfo=timezone.utc) if request.GET.get("end"): - end = datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S").astimezone( - timezone.utc + end = datetime.strptime(request.GET["end"], "%y_%m_%d_%H_%M_%S").replace( + tzinfo=timezone.utc ) except ValueError: return HttpResponseBadRequest( From e045edee606439bc62612d3e278e870369b1bf71 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 18:02:27 +0300 Subject: [PATCH 24/37] Update assert_called_with tests --- .../api/tests/viewsets/test_data_viewset.py | 30 +++++++++---------- .../test_submission_review_viewset.py | 20 ++++++------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/onadata/apps/api/tests/viewsets/test_data_viewset.py b/onadata/apps/api/tests/viewsets/test_data_viewset.py index 7adadc8b31..223925aae3 100644 --- a/onadata/apps/api/tests/viewsets/test_data_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_data_viewset.py @@ -1806,11 +1806,11 @@ def test_deletion_of_bulk_submissions(self, send_message_mock): ) self.assertTrue(send_message_mock.called) send_message_mock.assert_called_with( - [str(i.pk) for i in records_to_be_deleted], - formid, - XFORM, - request.user, - SUBMISSION_DELETED, + instance_id=[str(i.pk) for i in records_to_be_deleted], + target_id=formid, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, ) self.xform.refresh_from_db() current_count = self.xform.instances.filter(deleted_at=None).count() @@ -1906,11 +1906,11 @@ def test_permanent_deletions_bulk_submissions(self, send_message_mock): ) self.assertTrue(send_message_mock.called) send_message_mock.assert_called_with( - [str(i.pk) for i in records_to_be_deleted], - formid, - XFORM, - request.user, - SUBMISSION_DELETED, + instance_id=[str(i.pk) for i in records_to_be_deleted], + target_id=formid, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, ) self.xform.refresh_from_db() current_count = self.xform.num_of_submissions @@ -2059,11 +2059,11 @@ def test_delete_submissions(self, send_message_mock): ) self.assertTrue(send_message_mock.called) send_message_mock.assert_called_with( - [str(i.pk) for i in deleted_instances_subset], - formid, - XFORM, - request.user, - SUBMISSION_DELETED, + instance_id=[str(i.pk) for i in deleted_instances_subset], + target_id=formid, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_DELETED, ) # Test that num of submissions for the form is successfully updated diff --git a/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py b/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py index c442439530..12c146b970 100644 --- a/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_submission_review_viewset.py @@ -67,11 +67,11 @@ def test_submission_review_create(self, mock_send_message): # sends message upon saving the submission review self.assertTrue(mock_send_message.called) mock_send_message.assert_called_with( - submission_review.id, - submission_review.instance.xform.id, - XFORM, - submission_review.created_by, - SUBMISSION_REVIEWED, + instance_id=submission_review.instance_id, + target_id=submission_review.instance.xform.id, + target_type=XFORM, + user=submission_review.created_by, + message_verb=SUBMISSION_REVIEWED, ) @patch("onadata.apps.api.viewsets.submission_review_viewset.send_message") @@ -102,11 +102,11 @@ def test_bulk_create_submission_review(self, mock_send_message): # sends message upon saving the submission review self.assertTrue(mock_send_message.called) mock_send_message.assert_called_with( - [s.id for s in self.xform.instances.all()], - self.xform.id, - XFORM, - request.user, - SUBMISSION_REVIEWED, + instance_id=[s.id for s in self.xform.instances.all()], + target_id=self.xform.id, + target_type=XFORM, + user=request.user, + message_verb=SUBMISSION_REVIEWED, ) for item in response.data: # the note should match what we provided From 0d3de8f211e52eb9897dabaf5e5d256222abb724 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Mon, 25 Mar 2024 18:46:50 +0300 Subject: [PATCH 25/37] Update assert_called_with tests --- onadata/apps/api/tests/viewsets/test_xform_viewset.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/onadata/apps/api/tests/viewsets/test_xform_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_viewset.py index 9b9c385839..d125165ef9 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_viewset.py @@ -215,7 +215,11 @@ def test_replace_form_with_external_choices(self, mock_send_message): # send message upon form update self.assertTrue(mock_send_message.called) mock_send_message.assert_called_with( - self.xform.id, self.xform.id, XFORM, request.user, FORM_UPDATED + instance_id=self.xform.id, + target_id=self.xform.id, + target_type=XFORM, + user=request.user, + message_verb=FORM_UPDATED, ) def test_form_publishing_using_invalid_text_xls_form(self): From 35d7a7784a0b465b6ef2781a6e2d1f4d6462d8bf Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 26 Mar 2024 19:35:53 +0300 Subject: [PATCH 26/37] Django 4.1: update dependencies --- requirements/base.pip | 10 +++++----- requirements/dev.pip | 14 ++++++-------- setup.cfg | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/requirements/base.pip b/requirements/base.pip index d0ae022f23..32bf9f4fff 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -41,9 +41,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.69 +boto3==1.34.70 # via dataflows-tabulator -botocore==1.34.69 +botocore==1.34.70 # via # boto3 # s3transfer @@ -100,7 +100,7 @@ deprecated==1.2.14 # via onadata dict2xml==1.7.5 # via onadata -django==4.0 +django==4.1.13 # via # django-activity-stream # django-cors-headers @@ -135,7 +135,7 @@ django-guardian==2.4.0 # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==2.1.0 +django-oauth-toolkit==2.3.0 # via onadata django-ordered-model==3.7.4 # via onadata @@ -149,7 +149,7 @@ django-render-block==0.9.2 # via django-templated-email django-reversion==5.0.12 # via onadata -django-taggit==4.0.0 +django-taggit==5.0.1 # via onadata django-templated-email==3.0.1 # via onadata diff --git a/requirements/dev.pip b/requirements/dev.pip index 8f3db50884..cb40371530 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -49,9 +49,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.69 +boto3==1.34.70 # via dataflows-tabulator -botocore==1.34.69 +botocore==1.34.70 # via # boto3 # s3transfer @@ -118,7 +118,7 @@ dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -django==4.0 +django==4.1.13 # via # django-activity-stream # django-cors-headers @@ -156,7 +156,7 @@ django-guardian==2.4.0 # onadata django-nose==1.4.7 # via onadata -django-oauth-toolkit==2.1.0 +django-oauth-toolkit==2.3.0 # via onadata django-ordered-model==3.7.4 # via onadata @@ -170,7 +170,7 @@ django-render-block==0.9.2 # via django-templated-email django-reversion==5.0.12 # via onadata -django-taggit==4.0.0 +django-taggit==5.0.1 # via onadata django-templated-email==3.0.1 # via onadata @@ -208,7 +208,7 @@ et-xmlfile==1.1.0 # via openpyxl executing==2.0.1 # via stack-data -filelock==3.13.1 +filelock==3.13.3 # via virtualenv flake8==5.0.4 # via @@ -305,8 +305,6 @@ mccabe==0.7.0 # flake8 # prospector # pylint -mock==5.1.0 - # via -r requirements/dev.in modilabs-python-utils==0.1.5 # via onadata monotonic==1.6 diff --git a/setup.cfg b/setup.cfg index 2ccf9ad228..7f27898ffd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,7 +26,7 @@ tests_require = httmock requests-mock install_requires = - Django==4.0,<5 + Django==4.1.13 django-guardian django-registration-redux django-templated-email From e8d5fdf478675116c8d15419e9d0c7ebdeb4f65b Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 26 Mar 2024 23:38:18 +0300 Subject: [PATCH 27/37] Switch to the standard python library unittest.mock --- .../management/commands/test_delete_users.py | 15 +- .../tests/viewsets/test_briefcase_viewset.py | 617 +++++++++-------- .../api/tests/viewsets/test_charts_viewset.py | 25 +- .../viewsets/test_xform_submission_viewset.py | 8 +- onadata/apps/main/tests/test_form_show.py | 649 +++++++++++------- .../libs/tests/utils/test_api_export_tools.py | 131 ++-- 6 files changed, 846 insertions(+), 599 deletions(-) diff --git a/onadata/apps/api/tests/management/commands/test_delete_users.py b/onadata/apps/api/tests/management/commands/test_delete_users.py index ce975e13f5..2f1a484246 100644 --- a/onadata/apps/api/tests/management/commands/test_delete_users.py +++ b/onadata/apps/api/tests/management/commands/test_delete_users.py @@ -2,12 +2,17 @@ Test delete user management command. """ import sys -from unittest import mock -from six import StringIO -from django.contrib.auth.models import User +from unittest.mock import patch + +from django.contrib.auth import get_user_model from django.core.management import call_command -from onadata.apps.main.tests.test_base import TestBase + +from six import StringIO + from onadata.apps.api.management.commands.delete_users import get_user_object_stats +from onadata.apps.main.tests.test_base import TestBase + +User = get_user_model() class DeleteUserTest(TestBase): @@ -35,7 +40,7 @@ def test_delete_users_with_input(self): with self.assertRaises(User.DoesNotExist): User.objects.get(email="bruce@gmail.com") - @mock.patch("onadata.apps.api.management.commands.delete_users.input") + @patch("onadata.apps.api.management.commands.delete_users.input") def test_delete_users_no_input(self, mock_input): # pylint: disable=R0201 """ Test that when user_input is not provided, diff --git a/onadata/apps/api/tests/viewsets/test_briefcase_viewset.py b/onadata/apps/api/tests/viewsets/test_briefcase_viewset.py index 0e8b60f6f2..b43bd6b517 100644 --- a/onadata/apps/api/tests/viewsets/test_briefcase_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_briefcase_viewset.py @@ -1,13 +1,18 @@ +# -*- coding: utf-8 -*- +""" +Test BriefcaseViewset +""" import codecs import os import shutil +from unittest.mock import patch -import mock -from django.core.files.storage import get_storage_class from django.conf import settings +from django.core.files.storage import get_storage_class +from django.test import override_settings from django.urls import reverse from django.utils import timezone -from django.test import override_settings + from django_digest.test import DigestAuth from rest_framework.test import APIRequestFactory @@ -18,45 +23,50 @@ ) from onadata.apps.api.viewsets.xform_submission_viewset import XFormSubmissionViewSet from onadata.apps.api.viewsets.xform_viewset import XFormViewSet -from onadata.apps.logger.models import Instance -from onadata.apps.logger.models import XForm +from onadata.apps.logger.models import Instance, XForm NUM_INSTANCES = 4 storage = get_storage_class()() def ordered_instances(xform): - return Instance.objects.filter(xform=xform).order_by('id') + return Instance.objects.filter(xform=xform).order_by("id") class TestBriefcaseViewSet(test_abstract_viewset.TestAbstractViewSet): + """ + Test BriefcaseViewset + """ def setUp(self): super(test_abstract_viewset.TestAbstractViewSet, self).setUp() self.factory = APIRequestFactory() self._login_user_and_profile() - self.login_username = 'bob' - self.login_password = 'bobbob' + self.login_username = "bob" + self.login_password = "bobbob" self.maxDiff = None self.form_def_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'transportation.xml') + self.main_directory, "fixtures", "transportation", "transportation.xml" + ) self._submission_list_url = reverse( - 'view-submission-list', kwargs={'username': self.user.username}) + "view-submission-list", kwargs={"username": self.user.username} + ) self._submission_url = reverse( - 'submissions', kwargs={'username': self.user.username}) + "submissions", kwargs={"username": self.user.username} + ) self._download_submission_url = reverse( - 'view-download-submission', - kwargs={'username': self.user.username}) + "view-download-submission", kwargs={"username": self.user.username} + ) self._form_upload_url = reverse( - 'form-upload', kwargs={'username': self.user.username}) + "form-upload", kwargs={"username": self.user.username} + ) def _publish_xml_form(self, auth=None): - view = BriefcaseViewset.as_view({'post': 'create'}) + view = BriefcaseViewset.as_view({"post": "create"}) count = XForm.objects.count() - with codecs.open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with codecs.open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} auth = auth or DigestAuth(self.login_username, self.login_password) request = self.factory.post(self._form_upload_url, data=params) response = view(request, username=self.user.username) @@ -65,20 +75,29 @@ def _publish_xml_form(self, auth=None): response = view(request, username=self.user.username) self.assertEqual(XForm.objects.count(), count + 1) - self.assertContains( - response, "successfully published.", status_code=201) - self.xform = XForm.objects.order_by('pk').reverse()[0] + self.assertContains(response, "successfully published.", status_code=201) + self.xform = XForm.objects.order_by("pk").reverse()[0] def test_retrieve_encrypted_form_submissions(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) path = os.path.join( - settings.PROJECT_ROOT, "apps", "api", "tests", "fixtures", - "encrypted-form.xlsx") + settings.PROJECT_ROOT, + "apps", + "api", + "tests", + "fixtures", + "encrypted-form.xlsx", + ) submission_path = os.path.join( - settings.PROJECT_ROOT, "apps", "api", "tests", "fixtures", - "encrypted-submission.xml") + settings.PROJECT_ROOT, + "apps", + "api", + "tests", + "fixtures", + "encrypted-submission.xml", + ) self._publish_xls_form_to_project(xlsform_path=path) - form = XForm.objects.filter(id_string='hh_survey2').first() + form = XForm.objects.filter(id_string="hh_survey2").first() self._make_submission(submission_path) # Ensure media_all_received is false on the submission @@ -88,29 +107,29 @@ def test_retrieve_encrypted_form_submissions(self): self.assertEqual(instance.total_media, 2) self.assertEqual( set(instance.get_expected_media()), - set(['submission.xml.enc', '6-seater-7-15_15_11-15_45_15.jpg.enc']) + set(["submission.xml.enc", "6-seater-7-15_15_11-15_45_15.jpg.enc"]), ) self.assertFalse(instance.media_all_received) # Ensure submission is not returned on the Briefcase viewset request = self.factory.get( - self._submission_list_url, - data={'formId': form.id_string}) + self._submission_list_url, data={"formId": form.id_string} + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 200) - self.assertEqual(response.data['instances'].count(), 0) + self.assertEqual(response.data["instances"].count(), 0) def test_view_submission_list(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -118,55 +137,65 @@ def test_view_submission_list(self): response = view(request, username=self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.main_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with codecs.open(submission_list_path, 'rb', encoding='utf-8') as f: + with codecs.open(submission_list_path, "rb", encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) def test_view_submission_list_token_auth(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() # use Token auth in self.extra request = self.factory.get( self._submission_list_url, - data={'formId': self.xform.id_string}, **self.extra) + data={"formId": self.xform.id_string}, + **self.extra, + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.main_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with codecs.open(submission_list_path, 'rb', encoding='utf-8') as f: + with codecs.open(submission_list_path, "rb", encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) def test_view_submission_list_w_xformid(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() self._submission_list_url = reverse( - 'view-submission-list', - kwargs={'xform_pk': self.xform.pk}) + "view-submission-list", kwargs={"xform_pk": self.xform.pk} + ) request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, xform_pk=self.xform.pk) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -174,30 +203,34 @@ def test_view_submission_list_w_xformid(self): response = view(request, xform_pk=self.xform.pk) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.main_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with codecs.open(submission_list_path, 'rb', encoding='utf-8') as f: + with codecs.open(submission_list_path, "rb", encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) def test_view_submission_list_w_projectid(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() self._submission_list_url = reverse( - 'view-submission-list', - kwargs={'project_pk': self.xform.project.pk}) + "view-submission-list", kwargs={"project_pk": self.xform.project.pk} + ) request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, project_pk=self.xform.project.pk) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -205,25 +238,29 @@ def test_view_submission_list_w_projectid(self): response = view(request, project_pk=self.xform.project.pk) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submissionList.xml') + self.main_directory, + "fixtures", + "transportation", + "view", + "submissionList.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) last_index = instances[instances.count() - 1].pk - with codecs.open(submission_list_path, 'rb', encoding='utf-8') as f: + with codecs.open(submission_list_path, "rb", encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) def test_view_submission_list_w_soft_deleted_submission(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() - uuid = 'f3d8dc65-91a6-4d0f-9e97-802128083390' + uuid = "f3d8dc65-91a6-4d0f-9e97-802128083390" # soft delete submission instance = Instance.objects.filter(uuid=uuid).first() @@ -231,8 +268,8 @@ def test_view_submission_list_w_soft_deleted_submission(self): instance.save() request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -243,20 +280,19 @@ def test_view_submission_list_w_soft_deleted_submission(self): # check that number of instances returned by response is equal to # number of instances that have not been soft deleted self.assertEqual( - response.data.get('instances').count(), - Instance.objects.filter( - xform=self.xform, deleted_at__isnull=True).count() + response.data.get("instances").count(), + Instance.objects.filter(xform=self.xform, deleted_at__isnull=True).count(), ) def test_view_submission_list_w_deleted_submission(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() - uuid = 'f3d8dc65-91a6-4d0f-9e97-802128083390' - Instance.objects.filter(uuid=uuid).order_by('id').delete() + uuid = "f3d8dc65-91a6-4d0f-9e97-802128083390" + Instance.objects.filter(uuid=uuid).order_by("id").delete() request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -264,28 +300,32 @@ def test_view_submission_list_w_deleted_submission(self): response = view(request, username=self.user.username) self.assertEqual(response.status_code, 200) submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submissionList-4.xml') + self.main_directory, + "fixtures", + "transportation", + "view", + "submissionList-4.xml", + ) instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES - 1) last_index = instances[instances.count() - 1].pk - with codecs.open(submission_list_path, 'rb', encoding='utf-8') as f: + with codecs.open(submission_list_path, "rb", encoding="utf-8") as f: expected_submission_list = f.read() - expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + expected_submission_list = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) - view = BriefcaseViewset.as_view({'get': 'retrieve'}) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': uuid} - params = {'formId': formId} - request = self.factory.get( - self._download_submission_url, data=params) + view = BriefcaseViewset.as_view({"get": "retrieve"}) + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": uuid} + ) + params = {"formId": formId} + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) auth = DigestAuth(self.login_username, self.login_password) @@ -294,16 +334,16 @@ def test_view_submission_list_w_deleted_submission(self): self.assertTrue(response.status_code, 404) def test_view_submission_list_OtherUser(self): - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() # alice cannot view bob's submissionList - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) - auth = DigestAuth('alice', 'bobbob') + auth = DigestAuth("alice", "bobbob") request = self.factory.get( - self._submission_list_url, - data={'formId': self.xform.id_string}) + self._submission_list_url, data={"formId": self.xform.id_string} + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) @@ -323,11 +363,11 @@ def get_last_index(xform, last_index=None): return get_last_index(xform) return 0 - view = BriefcaseViewset.as_view({'get': 'list'}) + view = BriefcaseViewset.as_view({"get": "list"}) self._publish_xml_form() self._make_submissions() - params = {'formId': self.xform.id_string} - params['numEntries'] = 2 + params = {"formId": self.xform.id_string} + params["numEntries"] = 2 instances = ordered_instances(self.xform) self.assertEqual(instances.count(), NUM_INSTANCES) @@ -336,9 +376,7 @@ def get_last_index(xform, last_index=None): last_expected_submission_list = "" for index in range(1, 5): auth = DigestAuth(self.login_username, self.login_password) - request = self.factory.get( - self._submission_list_url, - data=params) + request = self.factory.get(self._submission_list_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) @@ -346,176 +384,200 @@ def get_last_index(xform, last_index=None): self.assertEqual(response.status_code, 200) if index > 2: last_index = get_last_index(self.xform, last_index) - filename = 'submissionList-%s.xml' % index + filename = "submissionList-%s.xml" % index if index == 4: self.assertContains(response, last_expected_submission_list) continue # set cursor for second request - params['cursor'] = last_index + params["cursor"] = last_index submission_list_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', filename) - with codecs.open(submission_list_path, encoding='utf-8') as f: + self.main_directory, "fixtures", "transportation", "view", filename + ) + with codecs.open(submission_list_path, encoding="utf-8") as f: expected_submission_list = f.read() - last_expected_submission_list = expected_submission_list = \ - expected_submission_list.replace( - '{{resumptionCursor}}', '%s' % last_index) + last_expected_submission_list = ( + expected_submission_list + ) = expected_submission_list.replace( + "{{resumptionCursor}}", "%s" % last_index + ) self.assertContains(response, expected_submission_list) last_index += 2 def test_view_downloadSubmission(self): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} auth = DigestAuth(self.login_username, self.login_password) - request = self.factory.get( - self._download_submission_url, data=params) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_w_token_auth(self): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} # use Token auth in self.extra request = self.factory.get( - self._download_submission_url, data=params, **self.extra) + self._download_submission_url, data=params, **self.extra + ) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 200) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_w_xformid(self): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} auth = DigestAuth(self.login_username, self.login_password) self._download_submission_url = reverse( - 'view-download-submission', - kwargs={'xform_pk': self.xform.pk}) - request = self.factory.get( - self._download_submission_url, data=params) + "view-download-submission", kwargs={"xform_pk": self.xform.pk} + ) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, xform_pk=self.xform.pk) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) response = view(request, xform_pk=self.xform.pk) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_w_projectid(self): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} auth = DigestAuth(self.login_username, self.login_password) self._download_submission_url = reverse( - 'view-download-submission', - kwargs={'project_pk': self.xform.project.pk}) - request = self.factory.get( - self._download_submission_url, data=params) + "view-download-submission", kwargs={"project_pk": self.xform.project.pk} + ) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, project_pk=self.xform.project.pk) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) response = view(request, project_pk=self.xform.project.pk) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) - self.assertMultiLineEqual(response.content.decode('utf-8'), text) + self.assertMultiLineEqual(response.content.decode("utf-8"), text) def test_view_downloadSubmission_OtherUser(self): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} # alice cannot view bob's downloadSubmission - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) - auth = DigestAuth('alice', 'bobbob') - request = self.factory.get( - self._download_submission_url, data=params) + auth = DigestAuth("alice", "bobbob") + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) @@ -523,15 +585,15 @@ def test_view_downloadSubmission_OtherUser(self): self.assertEqual(response.status_code, 404) def test_publish_xml_form_OtherUser(self): - view = BriefcaseViewset.as_view({'post': 'create'}) + view = BriefcaseViewset.as_view({"post": "create"}) # deno cannot publish form to bob's account - alice_data = {'username': 'alice', 'email': 'alice@localhost.com'} + alice_data = {"username": "alice", "email": "alice@localhost.com"} self._create_user_profile(alice_data) count = XForm.objects.count() - with codecs.open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} - auth = DigestAuth('alice', 'bobbob') + with codecs.open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} + auth = DigestAuth("alice", "bobbob") request = self.factory.post(self._form_upload_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) @@ -541,13 +603,13 @@ def test_publish_xml_form_OtherUser(self): self.assertEqual(response.status_code, 403) def test_publish_xml_form_where_filename_is_not_id_string(self): - view = BriefcaseViewset.as_view({'post': 'create'}) + view = BriefcaseViewset.as_view({"post": "create"}) form_def_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'Transportation Form.xml') + self.main_directory, "fixtures", "transportation", "Transportation Form.xml" + ) count = XForm.objects.count() - with codecs.open(form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with codecs.open(form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} auth = DigestAuth(self.login_username, self.login_password) request = self.factory.post(self._form_upload_url, data=params) response = view(request, username=self.user.username) @@ -555,15 +617,14 @@ def test_publish_xml_form_where_filename_is_not_id_string(self): request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) self.assertEqual(XForm.objects.count(), count + 1) - self.assertContains( - response, "successfully published.", status_code=201) + self.assertContains(response, "successfully published.", status_code=201) def test_form_upload(self): - view = BriefcaseViewset.as_view({'post': 'create'}) + view = BriefcaseViewset.as_view({"post": "create"}) self._publish_xml_form() - with codecs.open(self.form_def_path, encoding='utf-8') as f: - params = {'form_def_file': f, 'dataFile': ''} + with codecs.open(self.form_def_path, encoding="utf-8") as f: + params = {"form_def_file": f, "dataFile": ""} auth = DigestAuth(self.login_username, self.login_password) request = self.factory.post(self._form_upload_url, data=params) response = view(request, username=self.user.username) @@ -573,12 +634,11 @@ def test_form_upload(self): self.assertEqual(response.status_code, 400) self.assertEqual( response.data, - {'message': u'Form with this id or SMS-keyword already exists.' - } + {"message": "Form with this id or SMS-keyword already exists."}, ) def test_upload_head_request(self): - view = BriefcaseViewset.as_view({'head': 'create'}) + view = BriefcaseViewset.as_view({"head": "create"}) auth = DigestAuth(self.login_username, self.login_password) request = self.factory.head(self._form_upload_url) @@ -587,28 +647,26 @@ def test_upload_head_request(self): request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 204) - self.assertTrue(response.has_header('X-OpenRosa-Version')) - self.assertTrue( - response.has_header('X-OpenRosa-Accept-Content-Length')) - self.assertTrue(response.has_header('Date')) + self.assertTrue(response.has_header("X-OpenRosa-Version")) + self.assertTrue(response.has_header("X-OpenRosa-Accept-Content-Length")) + self.assertTrue(response.has_header("Date")) def test_submission_with_instance_id_on_root_node(self): - view = XFormSubmissionViewSet.as_view({'post': 'create'}) + view = XFormSubmissionViewSet.as_view({"post": "create"}) self._publish_xml_form() - message = u"Successful submission." - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' - self.assertRaises( - Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) + message = "Successful submission." + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" + self.assertRaises(Instance.DoesNotExist, Instance.objects.get, uuid=instanceId) submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'submission.xml') + self.main_directory, "fixtures", "transportation", "view", "submission.xml" + ) count = Instance.objects.count() - with codecs.open(submission_path, encoding='utf-8') as f: - post_data = {'xml_submission_file': f} + with codecs.open(submission_path, encoding="utf-8") as f: + post_data = {"xml_submission_file": f} request = self.factory.post(self._submission_list_url, post_data) response = view(request) self.assertEqual(response.status_code, 401) - auth = DigestAuth('bob', 'bobbob') + auth = DigestAuth("bob", "bobbob") request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) self.assertContains(response, message, status_code=201) @@ -617,62 +675,67 @@ def test_submission_with_instance_id_on_root_node(self): def test_form_export_with_no_xlsform_returns_200(self): self._publish_xml_form() - self.view = XFormViewSet.as_view({'get': 'retrieve'}) + self.view = XFormViewSet.as_view({"get": "retrieve"}) xform = XForm.objects.get(id_string="transportation_2011_07_25") - request = self.factory.get('/', **self.extra) - response = self.view(request, pk=xform.pk, format='csv') + request = self.factory.get("/", **self.extra) + response = self.view(request, pk=xform.pk, format="csv") self.assertEqual(response.status_code, 200) - self.view = XFormViewSet.as_view({'get': 'form'}) - response = self.view(request, pk=xform.pk, format='xls') + self.view = XFormViewSet.as_view({"get": "form"}) + response = self.view(request, pk=xform.pk, format="xls") self.assertEqual(response.status_code, 404) - @mock.patch.object(BriefcaseViewset, 'get_object') + @patch.object(BriefcaseViewset, "get_object") def test_view_downloadSubmission_no_xmlns(self, mock_get_object): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - instance.xml = u'nonenoneuuid:5b2cc313-fc09-437e-8149-fcd32f695d41\n' # noqa + instance.xml = 'nonenoneuuid:5b2cc313-fc09-437e-8149-fcd32f695d41\n' # noqa mock_get_object.return_value = instance - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} auth = DigestAuth(self.login_username, self.login_password) - request = self.factory.get( - self._download_submission_url, data=params) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertNotIn( 'transportation id="transportation_2011_07_25"' ' instanceID="uuid:5b2cc313-fc09-437e-8149-fcd32f695d41"' f' submissionDate="{ instance.date_created.isoformat() }" ' 'xlmns="http://opendatakit.org/submission"', - text) + text, + ) self.assertContains(response, instanceId, status_code=200) with override_settings(SUPPORT_BRIEFCASE_SUBMISSION_DATE=False): - request = self.factory.get( - self._download_submission_url, data=params) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) @@ -683,40 +746,46 @@ def test_view_downloadSubmission_no_xmlns(self, mock_get_object): ' id="transportation_2011_07_25"' ' instanceID="uuid:5b2cc313-fc09-437e-8149-fcd32f695d41"' f' submissionDate="{ instance.date_created.isoformat() }"', - response.content.decode('utf-8')) + response.content.decode("utf-8"), + ) - @mock.patch.object(BriefcaseViewset, 'get_object') + @patch.object(BriefcaseViewset, "get_object") def test_view_downloadSubmission_multiple_nodes(self, mock_get_object): - view = BriefcaseViewset.as_view({'get': 'retrieve'}) + view = BriefcaseViewset.as_view({"get": "retrieve"}) self._publish_xml_form() self.maxDiff = None self._submit_transport_instance_w_attachment() - instanceId = u'5b2cc313-fc09-437e-8149-fcd32f695d41' + instanceId = "5b2cc313-fc09-437e-8149-fcd32f695d41" instance = Instance.objects.get(uuid=instanceId) - instance.xml = u'nonenoneuuid:5b2cc313-fc09-437e-8149-fcd32f695d41\n' # noqa + instance.xml = "nonenoneuuid:5b2cc313-fc09-437e-8149-fcd32f695d41\n" # noqa mock_get_object.return_value = instance - formId = u'%(formId)s[@version=null and @uiVersion=null]/' \ - u'%(formId)s[@key=uuid:%(instanceId)s]' % { - 'formId': self.xform.id_string, - 'instanceId': instanceId} - params = {'formId': formId} + formId = ( + "%(formId)s[@version=null and @uiVersion=null]/" + "%(formId)s[@key=uuid:%(instanceId)s]" + % {"formId": self.xform.id_string, "instanceId": instanceId} + ) + params = {"formId": formId} auth = DigestAuth(self.login_username, self.login_password) - request = self.factory.get( - self._download_submission_url, data=params) + request = self.factory.get(self._download_submission_url, data=params) response = view(request, username=self.user.username) self.assertEqual(response.status_code, 401) request.META.update(auth(request.META, response)) response = view(request, username=self.user.username) text = "uuid:%s" % instanceId download_submission_path = os.path.join( - self.main_directory, 'fixtures', 'transportation', - 'view', 'downloadSubmission.xml') - with codecs.open(download_submission_path, encoding='utf-8') as f: + self.main_directory, + "fixtures", + "transportation", + "view", + "downloadSubmission.xml", + ) + with codecs.open(download_submission_path, encoding="utf-8") as f: text = f.read() - for var in ((u'{{submissionDate}}', - instance.date_created.isoformat()), - (u'{{form_id}}', str(self.xform.id)), - (u'{{media_id}}', str(self.attachment.id))): + for var in ( + ("{{submissionDate}}", instance.date_created.isoformat()), + ("{{form_id}}", str(self.xform.id)), + ("{{media_id}}", str(self.attachment.id)), + ): text = text.replace(*var) self.assertContains(response, instanceId, status_code=200) diff --git a/onadata/apps/api/tests/viewsets/test_charts_viewset.py b/onadata/apps/api/tests/viewsets/test_charts_viewset.py index b334d5a1a1..21648cec9e 100644 --- a/onadata/apps/api/tests/viewsets/test_charts_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_charts_viewset.py @@ -1,22 +1,26 @@ +# -*- coding: utf-8 -*- +""" +Test ChartsViewSet. +""" import json import os -import mock +from unittest.mock import patch -from django.utils import timezone from django.core.cache import cache +from django.db.utils import DataError from django.test.utils import override_settings -from rest_framework.test import APIClient -from rest_framework.test import APIRequestFactory -from rest_framework.test import force_authenticate +from django.utils import timezone + +from rest_framework.test import APIClient, APIRequestFactory, force_authenticate + from onadata.apps.api.viewsets.charts_viewset import ChartsViewSet from onadata.apps.api.viewsets.merged_xform_viewset import MergedXFormViewSet -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.logger.models.instance import Instance -from django.db.utils import DataError +from onadata.apps.main.tests.test_base import TestBase +from onadata.libs.renderers.renderers import DecimalJSONRenderer +from onadata.libs.utils.cache_tools import XFORM_CHARTS from onadata.libs.utils.timing import calculate_duration from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.cache_tools import XFORM_CHARTS -from onadata.libs.renderers.renderers import DecimalJSONRenderer def raise_data_error(a): @@ -48,7 +52,6 @@ def raise_data_error(a): class TestChartsViewSet(TestBase): - def setUp(self): super(self.__class__, self).setUp() # publish tutorial form as it has all the different field types @@ -193,7 +196,7 @@ def test_get_on_date_field(self): self.assertEqual(response.data["field_name"], "date") self.assertEqual(response.data["data_type"], "time_based") - @mock.patch("onadata.libs.data.query._execute_query", side_effect=raise_data_error) + @patch("onadata.libs.data.query._execute_query", side_effect=raise_data_error) def test_get_on_date_field_with_invalid_data(self, mock_execute_query): data = {"field_name": "date"} request = self.factory.get("/charts", data) diff --git a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py index 9f7cbfb7b8..04c4667931 100644 --- a/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py +++ b/onadata/apps/api/tests/viewsets/test_xform_submission_viewset.py @@ -4,6 +4,7 @@ """ import os from builtins import open # pylint: disable=redefined-builtin +from unittest.mock import patch from django.conf import settings from django.contrib.auth.models import AnonymousUser @@ -11,7 +12,6 @@ from django.http import UnreadablePostError from django.test import TransactionTestCase -import mock import simplejson as json from django_digest.test import DigestAuth @@ -843,7 +843,7 @@ def test_floip_format_multiple_rows_instance(self): data_responses = [i[4] for i in json.loads(data)] self.assertTrue(any(i in data_responses for i in instance_json.values())) - @mock.patch( + @patch( "onadata.apps.api.viewsets.xform_submission_viewset.SubmissionSerializer" ) # noqa def test_post_submission_unreadable_post_error(self, MockSerializer): @@ -1232,7 +1232,7 @@ def test_post_submission_using_project_pk_while_authenticated(self): Instance.objects.filter(xform=self.xform).count(), count + 1 ) - @mock.patch.object(ServiceDefinition, "send") + @patch.object(ServiceDefinition, "send") def test_new_submission_sent_to_rapidpro(self, mock_send): """Submission created is sent to RapidPro""" rest_service = RestService.objects.create( @@ -1276,7 +1276,7 @@ def test_new_submission_sent_to_rapidpro(self, mock_send): instance = Instance.objects.all().order_by("-pk")[0] mock_send.assert_called_once_with(rest_service.service_url, instance) - @mock.patch.object(ServiceDefinition, "send") + @patch.object(ServiceDefinition, "send") def test_edit_submission_sent_to_rapidpro(self, mock_send): """Submission edited is sent to RapidPro""" rest_service = RestService.objects.create( diff --git a/onadata/apps/main/tests/test_form_show.py b/onadata/apps/main/tests/test_form_show.py index 403e591fce..7a5d586dc4 100644 --- a/onadata/apps/main/tests/test_form_show.py +++ b/onadata/apps/main/tests/test_form_show.py @@ -1,25 +1,37 @@ +# -*- coding: utf-8 -*- +""" +Test form views. +""" import os -from builtins import open from unittest import skip +from unittest.mock import patch -import mock from django.core.exceptions import MultipleObjectsReturned from django.core.files.base import ContentFile +from django.test.utils import override_settings from django.urls import reverse + from httmock import HTTMock -from django.test.utils import override_settings from onadata.apps.api.tests.mocked_data import enketo_urls_mock from onadata.apps.logger.models import XForm -from onadata.apps.logger.views import download_xlsform, download_jsonform, \ - download_xform, delete_xform +from onadata.apps.logger.views import ( + delete_xform, + download_jsonform, + download_xform, + download_xlsform, +) from onadata.apps.main.tests.test_base import TestBase -from onadata.apps.main.views import show, form_photos, update_xform, profile, \ - enketo_preview -from onadata.apps.viewer.views import export_list, map_view, data_export +from onadata.apps.main.views import ( + enketo_preview, + form_photos, + profile, + show, + update_xform, +) +from onadata.apps.viewer.views import data_export, export_list, map_view from onadata.libs.utils.logger_tools import publish_xml_form -from onadata.libs.utils.user_auth import get_user_default_project -from onadata.libs.utils.user_auth import http_auth_string +from onadata.libs.utils.user_auth import get_user_default_project, http_auth_string def raise_multiple_objects_returned_error(*args, **kwargs): @@ -27,15 +39,18 @@ def raise_multiple_objects_returned_error(*args, **kwargs): class TestFormShow(TestBase): + """ + Test form views. + """ def setUp(self): TestBase.setUp(self) self._create_user_and_login() self._publish_transportation_form() - self.url = reverse(show, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + self.url = reverse( + show, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) def test_show_form_name(self): response = self.client.get(self.url) @@ -59,135 +74,197 @@ def test_show_to_anon_if_public(self): def test_dl_xlsx_xlsform(self): self._publish_xlsx_file() - response = self.client.get(reverse(download_xlsform, kwargs={ - 'username': self.user.username, - 'id_string': 'exp_one' - })) + response = self.client.get( + reverse( + download_xlsform, + kwargs={"username": self.user.username, "id_string": "exp_one"}, + ) + ) self.assertEqual(response.status_code, 200) self.assertEqual( - response['Content-Disposition'], - "attachment; filename=exp_one.xlsx") + response["Content-Disposition"], "attachment; filename=exp_one.xlsx" + ) # test with unavailable id_string - response = self.client.get(reverse(download_xlsform, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string' - })) + response = self.client.get( + reverse( + download_xlsform, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + }, + ) + ) self.assertEqual(response.status_code, 404) def test_dl_xls_to_anon_if_public(self): self.xform.shared = True self.xform.save() - response = self.anon.get(reverse(download_xlsform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - })) + response = self.anon.get( + reverse( + download_xlsform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) + ) self.assertEqual(response.status_code, 200) def test_dl_xls_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': - http_auth_string(self.login_username, self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } - response = self.anon.get(reverse(download_xlsform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }), **extra) + response = self.anon.get( + reverse( + download_xlsform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ), + **extra + ) self.assertEqual(response.status_code, 200) def test_dl_json_to_anon_if_public(self): self.xform.shared = True self.xform.save() - response = self.anon.get(reverse(download_jsonform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - })) + response = self.anon.get( + reverse( + download_jsonform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) + ) self.assertEqual(response.status_code, 200) # test with unavailable id_string - response = self.anon.get(reverse(download_jsonform, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string' - })) + response = self.anon.get( + reverse( + download_jsonform, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + }, + ) + ) self.assertEqual(response.status_code, 404) def test_dl_jsonp_to_anon_if_public(self): self.xform.shared = True self.xform.save() - callback = 'jsonpCallback' - response = self.anon.get(reverse(download_jsonform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }), {'callback': callback}) - content = response.content.decode('utf-8') + callback = "jsonpCallback" + response = self.anon.get( + reverse( + download_jsonform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ), + {"callback": callback}, + ) + content = response.content.decode("utf-8") self.assertEqual(response.status_code, 200) - self.assertEqual(content.startswith(callback + '('), True) - self.assertEqual(content.endswith(')'), True) + self.assertEqual(content.startswith(callback + "("), True) + self.assertEqual(content.endswith(")"), True) def test_dl_json_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': - http_auth_string(self.login_username, self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } - response = self.anon.get(reverse(download_jsonform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }), **extra) + response = self.anon.get( + reverse( + download_jsonform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ), + **extra + ) self.assertEqual(response.status_code, 200) def test_dl_json_for_cors_options(self): - response = self.anon.options(reverse(download_jsonform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - })) - allowed_headers = ['Accept', 'Origin', 'X-Requested-With', - 'Authorization'] - control_headers = response['Access-Control-Allow-Headers'] - provided_headers = [h.strip() for h in control_headers.split(',')] + response = self.anon.options( + reverse( + download_jsonform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) + ) + allowed_headers = ["Accept", "Origin", "X-Requested-With", "Authorization"] + control_headers = response["Access-Control-Allow-Headers"] + provided_headers = [h.strip() for h in control_headers.split(",")] self.assertListEqual(allowed_headers, provided_headers) - self.assertEqual(response['Access-Control-Allow-Methods'], 'GET') - self.assertEqual(response['Access-Control-Allow-Origin'], '*') + self.assertEqual(response["Access-Control-Allow-Methods"], "GET") + self.assertEqual(response["Access-Control-Allow-Origin"], "*") def test_dl_xform_to_anon_if_public(self): self.xform.shared = True self.xform.save() - response = self.anon.get(reverse(download_xform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - })) + response = self.anon.get( + reverse( + download_xform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) + ) self.assertEqual(response.status_code, 200) def test_dl_xform_for_basic_auth(self): extra = { - 'HTTP_AUTHORIZATION': - http_auth_string(self.login_username, self.login_password) + "HTTP_AUTHORIZATION": http_auth_string( + self.login_username, self.login_password + ) } - response = self.anon.get(reverse(download_xform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }), **extra) + response = self.anon.get( + reverse( + download_xform, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ), + **extra + ) self.assertEqual(response.status_code, 200) def test_dl_xform_for_authenticated_non_owner(self): - self._create_user_and_login('alice', 'alice') - response = self.client.get(reverse(download_xform, kwargs={ - 'username': 'bob', - 'id_string': self.xform.id_string - })) + self._create_user_and_login("alice", "alice") + response = self.client.get( + reverse( + download_xform, + kwargs={"username": "bob", "id_string": self.xform.id_string}, + ) + ) self.assertEqual(response.status_code, 200) # test with unavailable id_string - response = self.client.get(reverse(download_xform, kwargs={ - 'username': 'bob', - 'id_string': 'random_id_string' - })) + response = self.client.get( + reverse( + download_xform, + kwargs={"username": "bob", "id_string": "random_id_string"}, + ) + ) self.assertEqual(response.status_code, 404) def test_show_private_if_shared_but_not_data(self): self.xform.shared = True self.xform.save() response = self.anon.get(self.url) - self.assertContains(response, 'PRIVATE') + self.assertContains(response, "PRIVATE") def test_show_link_if_shared_and_data(self): self.xform.project.shared = True @@ -197,75 +274,111 @@ def test_show_link_if_shared_and_data(self): self.xform.save() self._submit_transport_instance() response = self.anon.get(self.url) - self.assertContains(response, reverse(export_list, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': 'csv' - })) + self.assertContains( + response, + reverse( + export_list, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": "csv", + }, + ), + ) # assert contains .xlsx in url - self.assertContains(response, reverse(export_list, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': 'xlsx' - })) + self.assertContains( + response, + reverse( + export_list, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": "xlsx", + }, + ), + ) # assert shouldn't contain .xls in url - self.assertNotContains(response, reverse(export_list, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': 'xls' - })) + self.assertNotContains( + response, + reverse( + export_list, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": "xls", + }, + ), + ) def test_return_error_if_xform_not_found(self): - map_url = reverse(map_view, kwargs={ - 'username': self.user.username, - 'id_string': 'random_string' - }) + map_url = reverse( + map_view, + kwargs={"username": self.user.username, "id_string": "random_string"}, + ) response = self.client.get(map_url) self.assertEqual(response.status_code, 404) - map_url = reverse(data_export, kwargs={ - 'username': self.user.username, - 'id_string': 'random_string' - }) + map_url = reverse( + data_export, + kwargs={"username": self.user.username, "id_string": "random_string"}, + ) response = self.client.get(map_url) self.assertEqual(response.status_code, 404) def test_show_link_if_owner(self): self._submit_transport_instance() response = self.client.get(self.url) - self.assertContains(response, reverse(export_list, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': 'csv' - })) - self.assertContains(response, reverse(export_list, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string, - 'export_type': 'xlsx' - })) - self.assertNotContains(response, reverse(map_view, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - })) + self.assertContains( + response, + reverse( + export_list, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": "csv", + }, + ), + ) + self.assertContains( + response, + reverse( + export_list, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + "export_type": "xlsx", + }, + ), + ) + self.assertNotContains( + response, + reverse( + map_view, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ), + ) # check that a form with geopoints has the map url count = XForm.objects.count() self._publish_xls_file( - os.path.join( - os.path.dirname(__file__), "fixtures", "gps", "gps.xlsx")) + os.path.join(os.path.dirname(__file__), "fixtures", "gps", "gps.xlsx") + ) self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.latest('date_created') - - show_url = reverse(show, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) - map_url = reverse(map_view, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + self.xform = XForm.objects.latest("date_created") + + show_url = reverse( + show, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) + map_url = reverse( + map_view, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) response = self.client.get(show_url) # check that map url doesnt show before we have submissions self.assertNotContains(response, map_url) @@ -273,8 +386,12 @@ def test_show_link_if_owner(self): # make a submission self._make_submission( os.path.join( - os.path.dirname(__file__), "fixtures", "gps", "instances", - "gps_1980-01-23_20-52-08.xml") + os.path.dirname(__file__), + "fixtures", + "gps", + "instances", + "gps_1980-01-23_20-52-08.xml", + ) ) self.assertEqual(self.response.status_code, 201) # get new show view @@ -283,57 +400,62 @@ def test_show_link_if_owner(self): def test_user_sees_edit_btn(self): response = self.client.get(self.url) - self.assertContains(response, 'edit') + self.assertContains(response, "edit") def test_user_sees_settings(self): response = self.client.get(self.url) - self.assertContains(response, 'Settings') + self.assertContains(response, "Settings") def test_anon_no_edit_btn(self): self.xform.shared = True self.xform.save() response = self.anon.get(self.url) - self.assertNotContains(response, 'edit') + self.assertNotContains(response, "edit") def test_anon_no_toggle_data_share_btn(self): self.xform.shared = True self.xform.save() response = self.anon.get(self.url) - self.assertNotContains(response, 'PUBLIC') - self.assertNotContains(response, 'PRIVATE') + self.assertNotContains(response, "PUBLIC") + self.assertNotContains(response, "PRIVATE") def test_show_add_sourc_doc_if_owner(self): response = self.client.get(self.url) - self.assertContains(response, 'Source document:') + self.assertContains(response, "Source document:") def test_show_add_supporting_docs_if_owner(self): response = self.client.get(self.url) - self.assertContains(response, 'Supporting document:') + self.assertContains(response, "Supporting document:") def test_show_add_supporting_media_if_owner(self): response = self.client.get(self.url) - self.assertContains(response, 'Media upload:') + self.assertContains(response, "Media upload:") def test_show_add_mapbox_layer_if_owner(self): response = self.client.get(self.url) - self.assertContains(response, 'JSONP url:') + self.assertContains(response, "JSONP url:") def test_hide_add_supporting_docs_if_not_owner(self): self.xform.shared = True self.xform.save() response = self.anon.get(self.url) - self.assertNotContains(response, 'Upload') + self.assertNotContains(response, "Upload") def test_load_photo_page(self): - response = self.client.get(reverse(form_photos, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string})) + response = self.client.get( + reverse( + form_photos, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) + ) self.assertEqual(response.status_code, 200) def test_load_from_uuid(self): self.xform = XForm.objects.get(pk=self.xform.id) - response = self.client.get(reverse(show, kwargs={ - 'uuid': self.xform.uuid})) + response = self.client.get(reverse(show, kwargs={"uuid": self.xform.uuid})) self.assertRedirects(response, self.url) def test_xls_replace_markup(self): @@ -343,56 +465,60 @@ def test_xls_replace_markup(self): # when we have 0 submissions, update markup exists self.xform.shared = True self.xform.save() - dashboard_url = reverse(profile, kwargs={ - 'username': 'bob' - }) + dashboard_url = reverse(profile, kwargs={"username": "bob"}) response = self.client.get(dashboard_url) - self.assertContains( - response, 'href="#replace-transportation_2011_07_25"') + self.assertContains(response, 'href="#replace-transportation_2011_07_25"') # a non owner can't see the markup response = self.anon.get(self.url) - self.assertNotContains( - response, 'href="#replace-transportation_2011_07_25"') + self.assertNotContains(response, 'href="#replace-transportation_2011_07_25"') # when we have a submission, we cant update the xls form self._submit_transport_instance() response = self.client.get(dashboard_url) - self.assertNotContains( - response, 'href="#replace-transportation_2011_07_25"') + self.assertNotContains(response, 'href="#replace-transportation_2011_07_25"') def test_non_owner_cannot_replace_form(self): """ Test that a non owner cannot replace a shared xls form """ - kwargs = { - 'username': self.user.username, - 'id_string': self.xform.id_string - } + kwargs = {"username": self.user.username, "id_string": self.xform.id_string} self.xform.shared = True self.xform.save() - request = self.factory.post('/') + request = self.factory.post("/") # create and login another user - self._create_user_and_login('peter', 'peter') + self._create_user_and_login("peter", "peter") request.user = self.user response = update_xform(request, **kwargs) self.assertEqual(response.status_code, 302) def test_replace_xform(self): - xform_update_url = reverse(update_xform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + xform_update_url = reverse( + update_xform, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) count = XForm.objects.count() - xls_path = os.path.join(self.this_directory, "fixtures", - "transportation", "transportation_updated.xlsx") - with open(xls_path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} + xls_path = os.path.join( + self.this_directory, + "fixtures", + "transportation", + "transportation_updated.xlsx", + ) + with open(xls_path, "rb") as xls_file: + post_data = {"xls_file": xls_file} self.client.post(xform_update_url, post_data) self.assertEqual(XForm.objects.count(), count) - self.xform = XForm.objects.order_by('id').reverse()[0] + self.xform = XForm.objects.order_by("id").reverse()[0] # look for the preferred_means question # which is only in the updated xls - is_updated_form = len([e.name for e in self.xform.survey_elements - if e.name == u'preferred_means']) > 0 + is_updated_form = ( + len( + [ + e.name + for e in self.xform.survey_elements + if e.name == "preferred_means" + ] + ) + > 0 + ) self.assertTrue(is_updated_form) def test_update_form_doesnt_truncate_to_50_chars(self): @@ -401,91 +527,111 @@ def test_update_form_doesnt_truncate_to_50_chars(self): self.this_directory, "fixtures", "transportation", - "transportation_with_long_id_string.xlsx") + "transportation_with_long_id_string.xlsx", + ) self._publish_xls_file_and_set_xform(xls_path) # Update the form - xform_update_url = reverse(update_xform, kwargs={ - 'username': self.user.username, - 'id_string': self.xform.id_string - }) + xform_update_url = reverse( + update_xform, + kwargs={"username": self.user.username, "id_string": self.xform.id_string}, + ) updated_xls_path = os.path.join( self.this_directory, "fixtures", "transportation", - "transportation_with_long_id_string_updated.xlsx") - with open(updated_xls_path, 'rb') as xls_file: - post_data = {'xls_file': xls_file} + "transportation_with_long_id_string_updated.xlsx", + ) + with open(updated_xls_path, "rb") as xls_file: + post_data = {"xls_file": xls_file} self.client.post(xform_update_url, post_data) # Count should stay the same self.assertEqual(XForm.objects.count(), count + 1) - self.xform = XForm.objects.order_by('id').reverse()[0] + self.xform = XForm.objects.order_by("id").reverse()[0] # look for the preferred_means question # which is only in the updated xls - is_updated_form = len([e.name for e in self.xform.survey_elements - if e.name == u'preferred_means']) > 0 + is_updated_form = ( + len( + [ + e.name + for e in self.xform.survey_elements + if e.name == "preferred_means" + ] + ) + > 0 + ) self.assertTrue(is_updated_form) def test_xform_delete(self): id_string = self.xform.id_string - form_exists = XForm.objects.filter( - user=self.user, id_string=id_string).count() == 1 + form_exists = ( + XForm.objects.filter(user=self.user, id_string=id_string).count() == 1 + ) self.assertTrue(form_exists) - xform_delete_url = reverse(delete_xform, kwargs={ - 'username': self.user.username, - 'id_string': id_string - }) + xform_delete_url = reverse( + delete_xform, + kwargs={"username": self.user.username, "id_string": id_string}, + ) self.client.post(xform_delete_url) - form_deleted = XForm.objects.filter( - user=self.user, id_string=id_string).count() == 0 + form_deleted = ( + XForm.objects.filter(user=self.user, id_string=id_string).count() == 0 + ) self.assertTrue(form_deleted) # test with unavailable id_string - xform_delete_url = reverse(delete_xform, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string' - }) + xform_delete_url = reverse( + delete_xform, + kwargs={"username": self.user.username, "id_string": "random_id_string"}, + ) response = self.client.post(xform_delete_url) self.assertEqual(response.status_code, 404) - @mock.patch('onadata.apps.logger.views.get_object_or_404', - side_effect=raise_multiple_objects_returned_error) + @patch( + "onadata.apps.logger.views.get_object_or_404", + side_effect=raise_multiple_objects_returned_error, + ) def test_delete_xforms_with_same_id_string_in_same_account( - self, mock_get_object_or_404): + self, mock_get_object_or_404 + ): id_string = self.xform.id_string - xform_delete_url = reverse(delete_xform, kwargs={ - 'username': self.user.username, - 'id_string': id_string - }) + xform_delete_url = reverse( + delete_xform, + kwargs={"username": self.user.username, "id_string": id_string}, + ) response = self.client.post(xform_delete_url) - form_deleted = XForm.objects.filter( - user=self.user, id_string=id_string).count() == 0 + form_deleted = ( + XForm.objects.filter(user=self.user, id_string=id_string).count() == 0 + ) self.assertTrue(form_deleted) self.assertEqual(response.status_code, 302) def test_non_owner_cant_delete_xform(self): id_string = self.xform.id_string - form_exists = XForm.objects.filter( - user=self.user, id_string=id_string).count() == 1 + form_exists = ( + XForm.objects.filter(user=self.user, id_string=id_string).count() == 1 + ) self.assertTrue(form_exists) - xform_delete_url = reverse(delete_xform, kwargs={ - 'username': self.user.username, - 'id_string': id_string - }) + xform_delete_url = reverse( + delete_xform, + kwargs={"username": self.user.username, "id_string": id_string}, + ) # save current user before we re-assign bob = self.user - self._create_user_and_login('alice', 'alice') + self._create_user_and_login("alice", "alice") self.client.post(xform_delete_url) - form_deleted = XForm.objects.filter( - user=bob, id_string=id_string).count() == 0 + form_deleted = XForm.objects.filter(user=bob, id_string=id_string).count() == 0 self.assertFalse(form_deleted) @override_settings(TESTING_MODE=False) def test_enketo_preview(self): with HTTMock(enketo_urls_mock): url = reverse( - enketo_preview, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + enketo_preview, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 302) @@ -494,42 +640,65 @@ def test_enketo_preview_works_on_shared_forms(self): self.xform.shared = True self.xform.save() url = reverse( - enketo_preview, kwargs={'username': self.user.username, - 'id_string': self.xform.id_string}) + enketo_preview, + kwargs={ + "username": self.user.username, + "id_string": self.xform.id_string, + }, + ) response = self.anon.get(url) self.assertEqual(response.status_code, 302) def test_enketo_preview_with_unavailable_id_string(self): - response = self.client.get(reverse(enketo_preview, kwargs={ - 'username': self.user.username, - 'id_string': 'random_id_string' - })) + response = self.client.get( + reverse( + enketo_preview, + kwargs={ + "username": self.user.username, + "id_string": "random_id_string", + }, + ) + ) self.assertEqual(response.status_code, 404) # TODO PLD disabling this test - @skip('Insensitivity is not enforced upon creation of id_strings.') + @skip("Insensitivity is not enforced upon creation of id_strings.") def test_form_urls_case_insensitive(self): - url = reverse(show, kwargs={ - 'username': self.user.username.upper(), - 'id_string': self.xform.id_string.upper() - }) + url = reverse( + show, + kwargs={ + "username": self.user.username.upper(), + "id_string": self.xform.id_string.upper(), + }, + ) response = self.client.get(url) self.assertEqual(response.status_code, 200) def test_publish_xml_xlsform_download(self): count = XForm.objects.count() path = os.path.join( - self.this_directory, '..', '..', 'api', 'tests', 'fixtures', - 'forms', 'contributions', 'contributions.xml') + self.this_directory, + "..", + "..", + "api", + "tests", + "fixtures", + "forms", + "contributions", + "contributions.xml", + ) f = open(path) xml_file = ContentFile(f.read()) f.close() - xml_file.name = 'contributions.xml' + xml_file.name = "contributions.xml" project = get_user_default_project(self.user) self.xform = publish_xml_form(xml_file, self.user, project) self.assertTrue(XForm.objects.count() > count) - response = self.client.get(reverse(download_xlsform, kwargs={ - 'username': self.user.username, - 'id_string': 'contributions' - }), follow=True) - self.assertContains(response, 'No XLS file for your form ') + response = self.client.get( + reverse( + download_xlsform, + kwargs={"username": self.user.username, "id_string": "contributions"}, + ), + follow=True, + ) + self.assertContains(response, "No XLS file for your form ") diff --git a/onadata/libs/tests/utils/test_api_export_tools.py b/onadata/libs/tests/utils/test_api_export_tools.py index fe8f09ff46..39784138c9 100644 --- a/onadata/libs/tests/utils/test_api_export_tools.py +++ b/onadata/libs/tests/utils/test_api_export_tools.py @@ -2,27 +2,30 @@ """ Test api_export_tools module. """ +import datetime from collections import OrderedDict, defaultdict +from unittest.mock import patch -import mock -import datetime -from google.oauth2.credentials import Credentials -from celery.backends.rpc import BacklogLimitExceeded from django.http import Http404 from django.test.utils import override_settings + +from celery.backends.rpc import BacklogLimitExceeded +from google.oauth2.credentials import Credentials from kombu.exceptions import OperationalError from rest_framework.request import Request from onadata.apps.logger.models import XForm -from onadata.apps.main.tests.test_base import TestBase from onadata.apps.main.models import TokenStorageModel +from onadata.apps.main.tests.test_base import TestBase from onadata.apps.viewer.models.export import Export, ExportConnectionError from onadata.libs.exceptions import ServiceUnavailable from onadata.libs.utils.api_export_tools import ( - get_async_response, get_existing_file_format, process_async_export, - response_for_format, + _get_google_credential, + get_async_response, + get_existing_file_format, get_metadata_format, - _get_google_credential + process_async_export, + response_for_format, ) from onadata.libs.utils.async_status import SUCCESSFUL, status_msg @@ -38,7 +41,7 @@ class TestApiExportTools(TestBase): "client_id": "client-id", "client_secret": "client-secret", "scopes": ["https://www.googleapis.com/auth/drive.file"], - "expiry": datetime.datetime(2016, 8, 18, 12, 43, 30, 316792) + "expiry": datetime.datetime(2016, 8, 18, 12, 43, 30, 316792), } def _create_old_export(self, xform, export_type, options, filename=None): @@ -48,22 +51,23 @@ def _create_old_export(self, xform, export_type, options, filename=None): export_type=export_type, options=options, filename=filename, - internal_status=Export.SUCCESSFUL).save() + internal_status=Export.SUCCESSFUL, + ).save() # pylint: disable=attribute-defined-outside-init - self.export = Export.objects.filter( - xform=xform, export_type=export_type)[0] + self.export = Export.objects.filter(xform=xform, export_type=export_type)[0] def test_get_google_credentials(self): """ Test create_async_export deletes credential when invalid """ - request = self.factory.get('/') + request = self.factory.get("/") request.user = self.user request.query_params = {} request.data = {} credential = self.google_credential - t = TokenStorageModel(id=self.user, - credential=Credentials(**credential, token=None)) + t = TokenStorageModel( + id=self.user, credential=Credentials(**credential, token=None) + ) t.save() self.assertFalse(t.credential.valid) response = _get_google_credential(request) @@ -71,7 +75,7 @@ def test_get_google_credentials(self): self.assertEqual(response.status_code, 302) self.assertEqual( response.url[:71], - 'https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=' + "https://accounts.google.com/o/oauth2/auth?response_type=code&client_id=", ) with self.assertRaises(TokenStorageModel.DoesNotExist): TokenStorageModel.objects.get(id=self.user) @@ -81,15 +85,17 @@ def test_get_google_credentials_valid(self): Test create_async_export does not get rid of valid credential """ - request = self.factory.get('/') + request = self.factory.get("/") request.user = self.user request.query_params = {} request.data = {} - self.google_credential['expiry'] = \ - datetime.datetime.utcnow() + datetime.timedelta(seconds=300) + self.google_credential[ + "expiry" + ] = datetime.datetime.utcnow() + datetime.timedelta(seconds=300) credential = self.google_credential - t = TokenStorageModel(id=self.user, - credential=Credentials(**credential, token="token")) + t = TokenStorageModel( + id=self.user, credential=Credentials(**credential, token="token") + ) t.save() self.assertTrue(t.credential.valid) credential = _get_google_credential(request) @@ -102,15 +108,14 @@ def test_process_async_export_creates_new_export(self): Test process_async_export creates a new export. """ self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user export_type = "csv" options = defaultdict(dict) - resp = process_async_export( - request, self.xform, export_type, options=options) + resp = process_async_export(request, self.xform, export_type, options=options) - self.assertIn('job_uuid', resp) + self.assertIn("job_uuid", resp) # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) @@ -122,24 +127,24 @@ def test_process_async_export_returns_existing_export(self): options = { "group_delimiter": "/", "remove_group_name": False, - "split_select_multiples": True + "split_select_multiples": True, } - request = Request(self.factory.post('/')) + request = Request(self.factory.post("/")) request.user = self.user export_type = "csv" self._create_old_export( - self.xform, export_type, options, filename="test_async_export") + self.xform, export_type, options, filename="test_async_export" + ) - resp = process_async_export( - request, self.xform, export_type, options=options) + resp = process_async_export(request, self.xform, export_type, options=options) - self.assertEqual(resp['job_status'], status_msg[SUCCESSFUL]) + self.assertEqual(resp["job_status"], status_msg[SUCCESSFUL]) self.assertIn("export_url", resp) # pylint: disable=invalid-name - @mock.patch('onadata.libs.utils.api_export_tools.AsyncResult') + @patch("onadata.libs.utils.api_export_tools.AsyncResult") @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_get_async_response_export_does_not_exist(self, AsyncResult): """ @@ -150,19 +155,19 @@ class MockAsyncResult(object): # pylint: disable=R0903 """Mock AsyncResult""" def __init__(self): - self.state = 'SUCCESS' + self.state = "SUCCESS" self.result = 1 AsyncResult.return_value = MockAsyncResult() self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user with self.assertRaises(Http404): - get_async_response('job_uuid', request, self.xform) + get_async_response("job_uuid", request, self.xform) # pylint: disable=invalid-name - @mock.patch('onadata.libs.utils.api_export_tools.AsyncResult') + @patch("onadata.libs.utils.api_export_tools.AsyncResult") @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_get_async_response_export_backlog_limit(self, AsyncResult): """ @@ -182,11 +187,11 @@ def state(self): AsyncResult.return_value = MockAsyncResult() self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user - result = get_async_response('job_uuid', request, self.xform) - self.assertEqual(result, {'job_status': 'PENDING'}) + result = get_async_response("job_uuid", request, self.xform) + self.assertEqual(result, {"job_status": "PENDING"}) def test_response_for_format(self): """ @@ -196,13 +201,12 @@ def test_response_for_format(self): xform = XForm.objects.filter().last() self.assertIsNotNone(xform) self.assertIsInstance(response_for_format(xform).data, dict) - self.assertIsInstance(response_for_format(xform, 'json').data, dict) - self.assertTrue( - hasattr(response_for_format(xform, 'xls').data, 'file')) + self.assertIsInstance(response_for_format(xform, "json").data, dict) + self.assertTrue(hasattr(response_for_format(xform, "xls").data, "file")) xform.xls.storage.delete(xform.xls.name) with self.assertRaises(Http404): - response_for_format(xform, 'xls') + response_for_format(xform, "xls") def test_get_metadata_format(self): """ @@ -210,16 +214,13 @@ def test_get_metadata_format(self): """ self._publish_xlsx_file() xform = XForm.objects.filter().last() - data_value = "xform_geojson {} {}".format( - xform.pk, xform.id_string) + data_value = "xform_geojson {} {}".format(xform.pk, xform.id_string) fmt = get_metadata_format(data_value) self.assertEqual("geojson", fmt) - data_value = "dataview_geojson {} {}".format( - xform.pk, xform.id_string) + data_value = "dataview_geojson {} {}".format(xform.pk, xform.id_string) fmt = get_metadata_format(data_value) self.assertEqual("geojson", fmt) - data_value = "xform {} {}".format( - xform.pk, xform.id_string) + data_value = "xform {} {}".format(xform.pk, xform.id_string) fmt = get_metadata_format(data_value) self.assertEqual(fmt, "csv") @@ -229,57 +230,57 @@ def test_get_existing_file_format(self): """ self._publish_xlsx_file() xform = XForm.objects.filter().last() - fmt = get_existing_file_format(xform.xls, 'xlsx') + fmt = get_existing_file_format(xform.xls, "xlsx") self.assertEqual("xlsx", fmt) # ensure it picks existing file extension regardless # of format passed in request params - fmt = get_existing_file_format(xform.xls, 'xls') + fmt = get_existing_file_format(xform.xls, "xls") self.assertEqual("xlsx", fmt) # pylint: disable=invalid-name - @mock.patch( - 'onadata.libs.utils.api_export_tools.viewer_task.create_async_export') + @patch("onadata.libs.utils.api_export_tools.viewer_task.create_async_export") def test_process_async_export_connection_error(self, mock_task): """ Test process_async_export creates a new export. """ mock_task.side_effect = ExportConnectionError self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user export_type = "csv" options = defaultdict(dict) with self.assertRaises(ServiceUnavailable): - process_async_export( - request, self.xform, export_type, options=options) + process_async_export(request, self.xform, export_type, options=options) # pylint: disable=invalid-name @override_settings(CELERY_TASK_ALWAYS_EAGER=True) - @mock.patch('onadata.libs.utils.api_export_tools.AsyncResult') + @patch("onadata.libs.utils.api_export_tools.AsyncResult") def test_get_async_response_connection_error(self, AsyncResult): """ Test get_async_response connection error. """ AsyncResult.side_effect = OperationalError self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user with self.assertRaises(ServiceUnavailable): - get_async_response('job_uuid', request, self.xform) + get_async_response("job_uuid", request, self.xform) - @mock.patch('onadata.libs.utils.api_export_tools.AsyncResult') + @patch("onadata.libs.utils.api_export_tools.AsyncResult") @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_get_async_response_when_result_changes_in_subsequent_calls( - self, AsyncResult): + self, AsyncResult + ): """ Test get_async_response export does not exist. """ class MockAsyncResult(object): # pylint: disable=R0903 """Mock AsyncResult""" - res = [1, {'PENDING': 'PENDING'}] + + res = [1, {"PENDING": "PENDING"}] def __init__(self): self.state = "PENDING" @@ -291,8 +292,8 @@ def result(self): AsyncResult.return_value = MockAsyncResult() self._publish_transportation_form_and_submit_instance() - request = self.factory.post('/') + request = self.factory.post("/") request.user = self.user - result = get_async_response('job_uuid', request, self.xform) - self.assertEqual(result, {'job_status': 'PENDING', 'progress': '1'}) + result = get_async_response("job_uuid", request, self.xform) + self.assertEqual(result, {"job_status": "PENDING", "progress": "1"}) From ebcb1a18c63f43d62c9341e6207bee57ea41932f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Mar 2024 00:07:25 +0300 Subject: [PATCH 28/37] Django 4.1: update dependecies --- requirements/azure.in | 2 +- requirements/azure.pip | 4 ++-- requirements/s3.in | 2 +- requirements/s3.pip | 8 ++++---- requirements/ses.in | 2 +- requirements/ses.pip | 8 ++++---- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/requirements/azure.in b/requirements/azure.in index fbae9b96a5..7c8f971e76 100644 --- a/requirements/azure.in +++ b/requirements/azure.in @@ -1,3 +1,3 @@ cryptography>=39.0.1 -django ==4.0,<5 +django ==4.1.13 django-storages[azure] diff --git a/requirements/azure.pip b/requirements/azure.pip index 91dfe77194..4996bf9ee8 100644 --- a/requirements/azure.pip +++ b/requirements/azure.pip @@ -4,7 +4,7 @@ # # pip-compile --output-file=requirements/azure.pip --strip-extras requirements/azure.in # -asgiref==3.7.2 +asgiref==3.8.1 # via django azure-core==1.30.1 # via @@ -22,7 +22,7 @@ cryptography==42.0.5 # via # -r requirements/azure.in # azure-storage-blob -django==4.0 +django==4.1.13 # via # -r requirements/azure.in # django-storages diff --git a/requirements/s3.in b/requirements/s3.in index 9dcb6b3af8..5232d5e4c0 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,3 +1,3 @@ boto3 -django ==4.0,<5 +django ==4.1.13 django-storages diff --git a/requirements/s3.pip b/requirements/s3.pip index 8e68f1a5fa..5dfa1a77c8 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -4,15 +4,15 @@ # # pip-compile --output-file=requirements/s3.pip --strip-extras requirements/s3.in # -asgiref==3.7.2 +asgiref==3.8.1 # via django -boto3==1.34.66 +boto3==1.34.71 # via -r requirements/s3.in -botocore==1.34.66 +botocore==1.34.71 # via # boto3 # s3transfer -django==4.0 +django==4.1.13 # via # -r requirements/s3.in # django-storages diff --git a/requirements/ses.in b/requirements/ses.in index d0a01aa09d..e62720a74b 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,3 +1,3 @@ boto -django ==4.0,<5 +django ==4.1.13 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index 5162be5bb4..57f2c90dab 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -4,17 +4,17 @@ # # pip-compile --output-file=requirements/ses.pip --strip-extras requirements/ses.in # -asgiref==3.7.2 +asgiref==3.8.1 # via django boto==2.49.0 # via -r requirements/ses.in -boto3==1.34.66 +boto3==1.34.71 # via django-ses -botocore==1.34.66 +botocore==1.34.71 # via # boto3 # s3transfer -django==4.0 +django==4.1.13 # via # -r requirements/ses.in # django-ses From ee5f61d44ec72a57e345e967099fcb85ce95f426 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Wed, 27 Mar 2024 11:37:46 +0300 Subject: [PATCH 29/37] Django 4.2: update dependecies --- requirements/azure.in | 2 +- requirements/azure.pip | 2 +- requirements/base.in | 12 ++++++------ requirements/base.pip | 36 ++++++++++++++++++------------------ requirements/dev.pip | 36 ++++++++++++++++++------------------ requirements/s3.in | 2 +- requirements/s3.pip | 2 +- requirements/ses.in | 2 +- requirements/ses.pip | 2 +- setup.cfg | 7 ++++--- 10 files changed, 52 insertions(+), 51 deletions(-) diff --git a/requirements/azure.in b/requirements/azure.in index 7c8f971e76..88168553d4 100644 --- a/requirements/azure.in +++ b/requirements/azure.in @@ -1,3 +1,3 @@ cryptography>=39.0.1 -django ==4.1.13 +django>=4.2.11,<5 django-storages[azure] diff --git a/requirements/azure.pip b/requirements/azure.pip index 4996bf9ee8..eb6756a559 100644 --- a/requirements/azure.pip +++ b/requirements/azure.pip @@ -22,7 +22,7 @@ cryptography==42.0.5 # via # -r requirements/azure.in # azure-storage-blob -django==4.1.13 +django==4.2.11 # via # -r requirements/azure.in # django-storages diff --git a/requirements/base.in b/requirements/base.in index b9286956c0..0755924497 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -2,10 +2,10 @@ -e . # installed from Git --e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest --e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient --e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc +git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest +git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest +git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router +git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip +git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient +git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc -e git+https://github.com/onaio/savreaderwriter.git@fix-pep-440-issues#egg=savreaderwriter diff --git a/requirements/base.pip b/requirements/base.pip index 32bf9f4fff..d3f695e47b 100644 --- a/requirements/base.pip +++ b/requirements/base.pip @@ -4,18 +4,6 @@ # # pip-compile --output-file=requirements/base.pip --strip-extras requirements/base.in # --e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest - # via -r requirements/base.in --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router - # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc - # via -r requirements/base.in --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip - # via -r requirements/base.in --e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest - # via -r requirements/base.in --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient - # via -r requirements/base.in -e git+https://github.com/onaio/savreaderwriter.git@fix-pep-440-issues#egg=savreaderwriter # via -r requirements/base.in alabaster==0.7.16 @@ -41,9 +29,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.70 +boto3==1.34.71 # via dataflows-tabulator -botocore==1.34.70 +botocore==1.34.71 # via # boto3 # s3transfer @@ -100,7 +88,7 @@ deprecated==1.2.14 # via onadata dict2xml==1.7.5 # via onadata -django==4.1.13 +django==4.2.11 # via # django-activity-stream # django-cors-headers @@ -127,12 +115,16 @@ django-csp==3.8 # via onadata django-debug-toolbar==4.3.0 # via onadata -django-filter==23.5 +django-digest @ git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92 + # via -r requirements/base.in +django-filter==24.1 # via onadata django-guardian==2.4.0 # via # djangorestframework-guardian # onadata +django-multidb-router @ git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52 + # via -r requirements/base.in django-nose==1.4.7 # via onadata django-oauth-toolkit==2.3.0 @@ -253,6 +245,8 @@ oauthlib==3.2.2 # via # django-oauth-toolkit # requests-oauthlib +ona-oidc @ git+https://github.com/onaio/ona-oidc.git@pytz-deprecated + # via -r requirements/base.in openpyxl==3.0.9 # via # dataflows-tabulator @@ -270,14 +264,16 @@ prompt-toolkit==3.0.43 # via click-repl psycopg2-binary==2.9.9 # via onadata -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth pycparser==2.21 # via cffi +pyfloip @ git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d + # via -r requirements/base.in pygments==2.17.2 # via sphinx pyjwt==2.8.0 @@ -298,6 +294,10 @@ python-dateutil==2.9.0.post0 # fleming # onadata # tableschema +python-digest @ git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36 + # via -r requirements/base.in +python-json2xlsclient @ git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0 + # via -r requirements/base.in python-memcached==1.62 # via onadata pytz==2024.1 diff --git a/requirements/dev.pip b/requirements/dev.pip index cb40371530..28137f7906 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -4,18 +4,6 @@ # # pip-compile --output-file=requirements/dev.pip --strip-extras requirements/dev.in # --e git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92#egg=django-digest - # via -r requirements/base.in --e git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52#egg=django-multidb-router - # via -r requirements/base.in --e git+https://github.com/onaio/ona-oidc.git@pytz-deprecated#egg=ona-oidc - # via -r requirements/base.in --e git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d#egg=pyfloip - # via -r requirements/base.in --e git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36#egg=python-digest - # via -r requirements/base.in --e git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0#egg=python-json2xlsclient - # via -r requirements/base.in -e git+https://github.com/onaio/savreaderwriter.git@fix-pep-440-issues#egg=savreaderwriter # via -r requirements/base.in alabaster==0.7.16 @@ -49,9 +37,9 @@ backoff==1.10.0 # via analytics-python billiard==4.2.0 # via celery -boto3==1.34.70 +boto3==1.34.71 # via dataflows-tabulator -botocore==1.34.70 +botocore==1.34.71 # via # boto3 # s3transfer @@ -118,7 +106,7 @@ dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -django==4.1.13 +django==4.2.11 # via # django-activity-stream # django-cors-headers @@ -146,14 +134,18 @@ django-csp==3.8 # via onadata django-debug-toolbar==4.3.0 # via onadata +django-digest @ git+https://github.com/onaio/django-digest.git@6bf61ec08502fd3545d4f2c0838b6cb15e7ffa92 + # via -r requirements/base.in django-extensions==3.2.3 # via -r requirements/dev.in -django-filter==23.5 +django-filter==24.1 # via onadata django-guardian==2.4.0 # via # djangorestframework-guardian # onadata +django-multidb-router @ git+https://github.com/onaio/django-multidb-router.git@f711368180d58eef87eda54fadfd5f8355623d52 + # via -r requirements/base.in django-nose==1.4.7 # via onadata django-oauth-toolkit==2.3.0 @@ -319,6 +311,8 @@ oauthlib==3.2.2 # via # django-oauth-toolkit # requests-oauthlib +ona-oidc @ git+https://github.com/onaio/ona-oidc.git@pytz-deprecated + # via -r requirements/base.in openpyxl==3.0.9 # via # dataflows-tabulator @@ -360,11 +354,11 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth pycodestyle==2.9.1 # via @@ -378,6 +372,8 @@ pyflakes==2.5.0 # via # flake8 # prospector +pyfloip @ git+https://github.com/onaio/floip-py.git@3c980eb184069ae7c3c9136b18441978237cd41d + # via -r requirements/base.in pygments==2.17.2 # via # ipython @@ -422,6 +418,10 @@ python-dateutil==2.9.0.post0 # fleming # onadata # tableschema +python-digest @ git+https://github.com/onaio/python-digest.git@08267ca8afc1a52f91352ebb5385e8e6d074fc36 + # via -r requirements/base.in +python-json2xlsclient @ git+https://github.com/onaio/python-json2xlsclient.git@62b4645f7b4f2684421a13ce98da0331a9dd66a0 + # via -r requirements/base.in python-memcached==1.62 # via onadata pytz==2024.1 diff --git a/requirements/s3.in b/requirements/s3.in index 5232d5e4c0..936c09c17a 100644 --- a/requirements/s3.in +++ b/requirements/s3.in @@ -1,3 +1,3 @@ boto3 -django ==4.1.13 +django>=4.2.11,<5 django-storages diff --git a/requirements/s3.pip b/requirements/s3.pip index 5dfa1a77c8..fe691e6de3 100644 --- a/requirements/s3.pip +++ b/requirements/s3.pip @@ -12,7 +12,7 @@ botocore==1.34.71 # via # boto3 # s3transfer -django==4.1.13 +django==4.2.11 # via # -r requirements/s3.in # django-storages diff --git a/requirements/ses.in b/requirements/ses.in index e62720a74b..3e46b26f84 100644 --- a/requirements/ses.in +++ b/requirements/ses.in @@ -1,3 +1,3 @@ boto -django ==4.1.13 +django>=4.2.11,<5 django-ses diff --git a/requirements/ses.pip b/requirements/ses.pip index 57f2c90dab..1ca64618da 100644 --- a/requirements/ses.pip +++ b/requirements/ses.pip @@ -14,7 +14,7 @@ botocore==1.34.71 # via # boto3 # s3transfer -django==4.1.13 +django==4.2.11 # via # -r requirements/ses.in # django-ses diff --git a/setup.cfg b/setup.cfg index 7f27898ffd..abb7e495ee 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,17 +1,18 @@ [metadata] name = onadata -version = 3.19.0 +version = 4.0.0 description = Collect Analyze and Share Data long_description = file: README.rst long_description_content_type = text/x-rst url = https://github.com/onaio/onadata author = Ona Systems Inc author_email = support@ona.io -license = Copyright (c) 2022 Ona Systems Inc All rights reserved +license = Copyright (c) 2024 Ona Systems Inc All rights reserved license_file = LICENSE classifiers = Development Status :: 5 - Production/Stable Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 project_urls = Documentation = https://api.ona.io/api Source = https://github.com/onaio/onadata @@ -26,7 +27,7 @@ tests_require = httmock requests-mock install_requires = - Django==4.1.13 + Django>=4.2.11,<5 django-guardian django-registration-redux django-templated-email From 5f28feabdfb9da0f9dd51753c463c5654749de47 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 2 Apr 2024 13:50:58 +0300 Subject: [PATCH 30/37] Django 4.2: Update __version__ --- onadata/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onadata/__init__.py b/onadata/__init__.py index 82441273fd..8fc1ffb3e5 100644 --- a/onadata/__init__.py +++ b/onadata/__init__.py @@ -6,7 +6,7 @@ """ from __future__ import absolute_import, unicode_literals -__version__ = "3.19.0" +__version__ = "4.0.0" # This will make sure the app is always imported when From 2d2ee8cc54a45010bbdb7f4d3a2502636580cd03 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Tue, 2 Apr 2024 13:57:15 +0300 Subject: [PATCH 31/37] Update pre-commit dependencies --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5feb00ad06..98b634345c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,12 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 24.3.0 hooks: - id: black From 800dd6d589a5a80ca13eddeece0817631089e121 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 12:31:13 +0300 Subject: [PATCH 32/37] Flaky tests - remove string encoding checks --- .../libs/tests/utils/test_export_builder.py | 178 +++++++++--------- 1 file changed, 89 insertions(+), 89 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index e4aeee65ef..589b684b84 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -75,7 +75,7 @@ class TestExportBuilder(TestBase): }, { "children/cartoons/name": "Flinstones", - "children/cartoons/why": "I like bam bam\u0107" + "children/cartoons/why": "I like bam bam\u0107", # throw in a unicode character }, ], @@ -130,7 +130,7 @@ class TestExportBuilder(TestBase): }, { "childrens_survey_with_a_very_lo/cartoons/name": "Flinstones", - "childrens_survey_with_a_very_lo/cartoons/why": "I like bam bam\u0107" + "childrens_survey_with_a_very_lo/cartoons/why": "I like bam bam\u0107", # throw in a unicode character }, ], @@ -613,12 +613,12 @@ def test_zipped_sav_export_with_date_field(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"expense_date") - self.assertEqual(rows[1][0], b"2013-01-03") - self.assertEqual(rows[0][1], b"A.gdate") - self.assertEqual(rows[1][1], b"2017-06-13") - self.assertEqual(rows[0][5], b"@_submission_time") - self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + self.assertEqual(rows[0][0], "expense_date") + self.assertEqual(rows[1][0], "2013-01-03") + self.assertEqual(rows[0][1], "A.gdate") + self.assertEqual(rows[1][1], "2017-06-13") + self.assertEqual(rows[0][5], "@_submission_time") + self.assertEqual(rows[1][5], "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -666,23 +666,23 @@ def test_zipped_sav_export_dynamic_select_multiple(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"sex") - self.assertEqual(rows[1][0], b"male") - self.assertEqual(rows[0][1], b"text") - self.assertEqual(rows[1][1], b"his") - self.assertEqual(rows[0][2], b"favorite_brand") - self.assertEqual(rows[1][2], b"Generic") - self.assertEqual(rows[0][3], b"name") - self.assertEqual(rows[1][3], b"Davis") - self.assertEqual(rows[0][4], b"brand_known") - self.assertEqual(rows[1][4], b"his Generic a") - self.assertEqual(rows[0][5], b"brand_known.$text") + self.assertEqual(rows[0][0], "sex") + self.assertEqual(rows[1][0], "male") + self.assertEqual(rows[0][1], "text") + self.assertEqual(rows[1][1], "his") + self.assertEqual(rows[0][2], "favorite_brand") + self.assertEqual(rows[1][2], "Generic") + self.assertEqual(rows[0][3], "name") + self.assertEqual(rows[1][3], "Davis") + self.assertEqual(rows[0][4], "brand_known") + self.assertEqual(rows[1][4], "his Generic a") + self.assertEqual(rows[0][5], "brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") + self.assertEqual(rows[0][6], "brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], b"brand_known.a") + self.assertEqual(rows[0][7], "brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], b"brand_known.b") + self.assertEqual(rows[0][8], "brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) @@ -759,16 +759,16 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], b"expensed") + self.assertEqual(rows[0][0], "expensed") self.assertEqual(rows[1][0], 1) # A/q1 1 - self.assertEqual(rows[0][1], b"A.q1") + self.assertEqual(rows[0][1], "A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], b"@_submission_time") - self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + self.assertEqual(rows[0][5], "@_submission_time") + self.assertEqual(rows[1][5], "2016-11-21 03:43:43") # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_field_different_groups(self): @@ -951,35 +951,35 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"expensed") - self.assertEqual(rows[1][0], b"1") + self.assertEqual(rows[0][0], "expensed") + self.assertEqual(rows[1][0], "1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][1], b"expensed.1") + self.assertEqual(rows[0][1], "expensed.1") self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][2], b"expensed.0") + self.assertEqual(rows[0][2], "expensed.0") self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[0][3], b"A.q1") - self.assertEqual(rows[1][3], b"1") + self.assertEqual(rows[0][3], "A.q1") + self.assertEqual(rows[1][3], "1") # ensure you get a numeric value for multiple select with choice # filters - self.assertEqual(rows[0][6], b"A.q2") - self.assertEqual(rows[1][6], b"1") + self.assertEqual(rows[0][6], "A.q2") + self.assertEqual(rows[1][6], "1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][4], b"A.q1.1") + self.assertEqual(rows[0][4], "A.q1.1") self.assertEqual(rows[1][4], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][5], b"A.q1.0") + self.assertEqual(rows[0][5], "A.q1.0") self.assertEqual(rows[1][5], 0) - self.assertEqual(rows[0][12], b"@_submission_time") - self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") + self.assertEqual(rows[0][12], "@_submission_time") + self.assertEqual(rows[1][12], "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1012,12 +1012,12 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0], b"1") + self.assertEqual(rows[1][0], "1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") + self.assertEqual(rows[1][6], "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1053,12 +1053,12 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0], b"2 09") + self.assertEqual(rows[1][0], "2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 2) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[1][2], b"09") - self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") + self.assertEqual(rows[1][2], "09") + self.assertEqual(rows[1][6], "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1165,10 +1165,10 @@ def test_zipped_sav_export_with_duplicate_column_name(self): rows = list(reader) # Check that columns are present - self.assertIn(b"Sport", rows[0]) + self.assertIn("Sport", rows[0]) # Check for sport in first 5 characters # because rows contains 'sport@d4b6' - self.assertIn(b"sport", [x[0:5] for x in rows[0]]) + self.assertIn("sport", [x[0:5] for x in rows[0]]) # pylint: disable=invalid-name def test_xlsx_export_works_with_unicode(self): @@ -2904,7 +2904,7 @@ def test_zipped_sav_has_submission_review_fields(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x.encode("utf-8") + x for x in [ "photo", "osm_road", @@ -2941,9 +2941,9 @@ def test_zipped_sav_has_submission_review_fields(self): ] ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], b"Rejected") - self.assertEqual(rows[1][30], b"Wrong Location") - self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") + self.assertEqual(rows[1][29], "Rejected") + self.assertEqual(rows[1][30], "Wrong Location") + self.assertEqual(rows[1][31], "2021-05-25T02:27:19") # pylint: disable=invalid-name def test_zipped_csv_export_with_osm_data(self): @@ -3041,7 +3041,7 @@ def test_zipped_sav_export_with_osm_data(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x.encode("utf-8") + x for x in [ "photo", "osm_road", @@ -3075,12 +3075,12 @@ def test_zipped_sav_export_with_osm_data(self): ] ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], b"1424308569120.jpg") - self.assertEqual(rows[1][1], b"OSMWay234134797.osm") - self.assertEqual(rows[1][2], b"23.708174238006087") - self.assertEqual(rows[1][4], b"tertiary") - self.assertEqual(rows[1][6], b"Patuatuli Road") - self.assertEqual(rows[1][13], b"kol") + self.assertEqual(rows[1][0], "1424308569120.jpg") + self.assertEqual(rows[1][1], "OSMWay234134797.osm") + self.assertEqual(rows[1][2], "23.708174238006087") + self.assertEqual(rows[1][4], "tertiary") + self.assertEqual(rows[1][6], "Patuatuli Road") + self.assertEqual(rows[1][13], "kol") def test_show_choice_labels(self): """ @@ -3607,52 +3607,52 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): expected_data = [ [ - b"gps", - b"@_gps_latitude", - b"@_gps_longitude", - b"@_gps_altitude", - b"@_gps_precision", - b"gps@52a9", - b"@_gps_latitude_52a9", - b"@_gps_longitude_52a9", - b"@_gps_altitude_52a9", - b"@_gps_precision_52a9", - b"instanceID", - b"@_id", - b"@_uuid", - b"@_submission_time", - b"@_index", - b"@_parent_table_name", - b"@_parent_index", - b"@_tags", - b"@_notes", - b"@_version", - b"@_duration", - b"@_submitted_by", + "gps", + "@_gps_latitude", + "@_gps_longitude", + "@_gps_altitude", + "@_gps_precision", + "gps@52a9", + "@_gps_latitude_52a9", + "@_gps_longitude_52a9", + "@_gps_altitude_52a9", + "@_gps_precision_52a9", + "instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", ], [ - b"4.0 36.1 5000 20", + "4.0 36.1 5000 20", 4.0, 36.1, 5000.0, 20.0, - b"1.0 36.1 2000 20", + "1.0 36.1 2000 20", 1.0, 36.1, 2000.0, 20.0, - b"", + "", None, - b"", - b"2016-11-21 03:42:43", + "", + "2016-11-21 03:42:43", 1.0, - b"", + "", -1.0, - b"", - b"", - b"", - b"", - b"", + "", + "", + "", + "", + "", ], ] with SavReader( From fb194995086cef8787211dcab35164f468adcd1f Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 13:04:01 +0300 Subject: [PATCH 33/37] Add quote to ppa:deadsnakes/ppa - address error adding the PPA --- docker/onadata-uwsgi/Dockerfile.ubuntu | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/onadata-uwsgi/Dockerfile.ubuntu b/docker/onadata-uwsgi/Dockerfile.ubuntu index b511802a4e..378936166d 100644 --- a/docker/onadata-uwsgi/Dockerfile.ubuntu +++ b/docker/onadata-uwsgi/Dockerfile.ubuntu @@ -44,10 +44,10 @@ ENV LC_CTYPE en_US.UTF-8 RUN dpkg-reconfigure locales # Add Deadsnake Repository -RUN add-apt-repository ppa:deadsnakes/ppa -y && apt-get update -q - # Install OnaData Dependencies -RUN apt-get install -y --no-install-recommends \ +RUN add-apt-repository 'ppa:deadsnakes/ppa' -y \ + && apt-get update -q \ + && apt-get install -y --no-install-recommends \ libproj-dev \ gdal-bin \ memcached \ @@ -111,5 +111,3 @@ EXPOSE 8000 USER onadata CMD ["/usr/local/bin/uwsgi", "--ini", "/uwsgi.ini"] - - From 7bad5206f29b6404eda98ccd383f743a6a816349 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 13:57:47 +0300 Subject: [PATCH 34/37] Flaky tests - string encoding --- .../libs/tests/utils/test_export_builder.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 589b684b84..85b61b9baa 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -759,16 +759,16 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], "expensed") + self.assertEqual(rows[0][0], b"expensed") self.assertEqual(rows[1][0], 1) # A/q1 1 - self.assertEqual(rows[0][1], "A.q1") + self.assertEqual(rows[0][1], b"A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], "@_submission_time") - self.assertEqual(rows[1][5], "2016-11-21 03:43:43") + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_field_different_groups(self): @@ -951,35 +951,35 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], "expensed") - self.assertEqual(rows[1][0], "1") + self.assertEqual(rows[0][0], b"expensed") + self.assertEqual(rows[1][0], b"1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][1], "expensed.1") + self.assertEqual(rows[0][1], b"expensed.1") self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][2], "expensed.0") + self.assertEqual(rows[0][2], b"expensed.0") self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[0][3], "A.q1") - self.assertEqual(rows[1][3], "1") + self.assertEqual(rows[0][3], b"A.q1") + self.assertEqual(rows[1][3], b"1") # ensure you get a numeric value for multiple select with choice # filters - self.assertEqual(rows[0][6], "A.q2") - self.assertEqual(rows[1][6], "1") + self.assertEqual(rows[0][6], b"A.q2") + self.assertEqual(rows[1][6], b"1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][4], "A.q1.1") + self.assertEqual(rows[0][4], b"A.q1.1") self.assertEqual(rows[1][4], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][5], "A.q1.0") + self.assertEqual(rows[0][5], b"A.q1.0") self.assertEqual(rows[1][5], 0) - self.assertEqual(rows[0][12], "@_submission_time") - self.assertEqual(rows[1][12], "2016-11-21 03:43:43") + self.assertEqual(rows[0][12], b"@_submission_time") + self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1012,12 +1012,12 @@ def test_zipped_sav_export_with_zero_padded_select_multiple_field(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0], "1") + self.assertEqual(rows[1][0], b"1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[1][6], "2016-11-21 03:43:43") + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1053,12 +1053,12 @@ def test_zipped_sav_export_with_values_split_select_multiple(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[1][0], "2 09") + self.assertEqual(rows[1][0], b"2 09") # expensed.1 is selected hence True, 1.00 or 1 in SPSS self.assertEqual(rows[1][1], 2) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[1][2], "09") - self.assertEqual(rows[1][6], "2016-11-21 03:43:43") + self.assertEqual(rows[1][2], b"09") + self.assertEqual(rows[1][6], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -2941,9 +2941,9 @@ def test_zipped_sav_has_submission_review_fields(self): ] ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], "Rejected") - self.assertEqual(rows[1][30], "Wrong Location") - self.assertEqual(rows[1][31], "2021-05-25T02:27:19") + self.assertEqual(rows[1][29], b"Rejected") + self.assertEqual(rows[1][30], b"Wrong Location") + self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") # pylint: disable=invalid-name def test_zipped_csv_export_with_osm_data(self): @@ -3075,12 +3075,12 @@ def test_zipped_sav_export_with_osm_data(self): ] ] self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], "1424308569120.jpg") - self.assertEqual(rows[1][1], "OSMWay234134797.osm") - self.assertEqual(rows[1][2], "23.708174238006087") - self.assertEqual(rows[1][4], "tertiary") - self.assertEqual(rows[1][6], "Patuatuli Road") - self.assertEqual(rows[1][13], "kol") + self.assertEqual(rows[1][0], b"1424308569120.jpg") + self.assertEqual(rows[1][1], b"OSMWay234134797.osm") + self.assertEqual(rows[1][2], b"23.708174238006087") + self.assertEqual(rows[1][4], b"tertiary") + self.assertEqual(rows[1][6], b"Patuatuli Road") + self.assertEqual(rows[1][13], b"kol") def test_show_choice_labels(self): """ From 5a0e3a7b8755a878da99004dcf9a2dcf050e6eb1 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 19:51:16 +0300 Subject: [PATCH 35/37] Flaky tests - string encoding --- .../libs/tests/utils/test_export_builder.py | 98 +++++++++---------- 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 85b61b9baa..7577a9381d 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -613,12 +613,12 @@ def test_zipped_sav_export_with_date_field(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], "expense_date") - self.assertEqual(rows[1][0], "2013-01-03") - self.assertEqual(rows[0][1], "A.gdate") - self.assertEqual(rows[1][1], "2017-06-13") - self.assertEqual(rows[0][5], "@_submission_time") - self.assertEqual(rows[1][5], "2016-11-21 03:43:43") + self.assertEqual(rows[0][0], b"expense_date") + self.assertEqual(rows[1][0], b"2013-01-03") + self.assertEqual(rows[0][1], b"A.gdate") + self.assertEqual(rows[1][1], b"2017-06-13") + self.assertEqual(rows[0][5], b"@_submission_time") + self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -666,23 +666,23 @@ def test_zipped_sav_export_dynamic_select_multiple(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], "sex") - self.assertEqual(rows[1][0], "male") - self.assertEqual(rows[0][1], "text") - self.assertEqual(rows[1][1], "his") - self.assertEqual(rows[0][2], "favorite_brand") - self.assertEqual(rows[1][2], "Generic") - self.assertEqual(rows[0][3], "name") - self.assertEqual(rows[1][3], "Davis") - self.assertEqual(rows[0][4], "brand_known") - self.assertEqual(rows[1][4], "his Generic a") - self.assertEqual(rows[0][5], "brand_known.$text") + self.assertEqual(rows[0][0], b"sex") + self.assertEqual(rows[1][0], b"male") + self.assertEqual(rows[0][1], b"text") + self.assertEqual(rows[1][1], b"his") + self.assertEqual(rows[0][2], b"favorite_brand") + self.assertEqual(rows[1][2], b"Generic") + self.assertEqual(rows[0][3], b"name") + self.assertEqual(rows[1][3], b"Davis") + self.assertEqual(rows[0][4], b"brand_known") + self.assertEqual(rows[1][4], b"his Generic a") + self.assertEqual(rows[0][5], b"brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], "brand_known.$favorite_brand") + self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], "brand_known.a") + self.assertEqual(rows[0][7], b"brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], "brand_known.b") + self.assertEqual(rows[0][8], b"brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) @@ -1165,10 +1165,10 @@ def test_zipped_sav_export_with_duplicate_column_name(self): rows = list(reader) # Check that columns are present - self.assertIn("Sport", rows[0]) + self.assertIn(b"Sport", rows[0]) # Check for sport in first 5 characters # because rows contains 'sport@d4b6' - self.assertIn("sport", [x[0:5] for x in rows[0]]) + self.assertIn(b"sport", [x[0:5] for x in rows[0]]) # pylint: disable=invalid-name def test_xlsx_export_works_with_unicode(self): @@ -2904,7 +2904,7 @@ def test_zipped_sav_has_submission_review_fields(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x + x.encode("utf-8") for x in [ "photo", "osm_road", @@ -3041,7 +3041,7 @@ def test_zipped_sav_export_with_osm_data(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x + x.encode("8") for x in [ "photo", "osm_road", @@ -3607,36 +3607,36 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): expected_data = [ [ - "gps", - "@_gps_latitude", - "@_gps_longitude", - "@_gps_altitude", - "@_gps_precision", - "gps@52a9", - "@_gps_latitude_52a9", - "@_gps_longitude_52a9", - "@_gps_altitude_52a9", - "@_gps_precision_52a9", - "instanceID", - "@_id", - "@_uuid", - "@_submission_time", - "@_index", - "@_parent_table_name", - "@_parent_index", - "@_tags", - "@_notes", - "@_version", - "@_duration", - "@_submitted_by", + b"gps", + b"@_gps_latitude", + b"@_gps_longitude", + b"@_gps_altitude", + b"@_gps_precision", + b"gps@52a9", + b"@_gps_latitude_52a9", + b"@_gps_longitude_52a9", + b"@_gps_altitude_52a9", + b"@_gps_precision_52a9", + b"instanceID", + b"@_id", + b"@_uuid", + b"@_submission_time", + b"@_index", + b"@_parent_table_name", + b"@_parent_index", + b"@_tags", + b"@_notes", + b"@_version", + b"@_duration", + b"@_submitted_by", ], [ - "4.0 36.1 5000 20", + b"4.0 36.1 5000 20", 4.0, 36.1, 5000.0, 20.0, - "1.0 36.1 2000 20", + b"1.0 36.1 2000 20", 1.0, 36.1, 2000.0, @@ -3644,7 +3644,7 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): "", None, "", - "2016-11-21 03:42:43", + b"2016-11-21 03:42:43", 1.0, "", -1.0, From 8cd8c57d4b2dc8d9ac103b7d66151f3f493cf168 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 21:38:24 +0300 Subject: [PATCH 36/37] Flaky tests - string encoding --- .../libs/tests/utils/test_export_builder.py | 81 ++++++++++--------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 7577a9381d..7edf2723bd 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -51,6 +51,11 @@ def _logger_fixture_path(*args): ) +def _str_if_bytes(val): + """Returns val as string if it is of type bytes otherwise returns bytes""" + return str(val, "utf-8") if isinstance(val, bytes) else val + + class TestExportBuilder(TestBase): """Test onadata.libs.utils.export_builder functions.""" @@ -2904,46 +2909,44 @@ def test_zipped_sav_has_submission_review_fields(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x.encode("utf-8") - for x in [ - "photo", - "osm_road", - "osm_building", - "fav_color", - "form_completed", - "meta.instanceID", - "@_id", - "@_uuid", - "@_submission_time", - "@_index", - "@_parent_table_name", - "@_review_comment", - f"@{REVIEW_DATE}", - "@_review_status", - "@_parent_index", - "@_tags", - "@_notes", - "@_version", - "@_duration", - "@_submitted_by", - "osm_road_ctr_lat", - "osm_road_ctr_lon", - "osm_road_highway", - "osm_road_lanes", - "osm_road_name", - "osm_road_way_id", - "osm_building_building", - "osm_building_building_levels", - "osm_building_ctr_lat", - "osm_building_ctr_lon", - "osm_building_name", - "osm_building_way_id", - ] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_review_comment", + f"@{REVIEW_DATE}", + "@_review_status", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", ] - self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][29], b"Rejected") - self.assertEqual(rows[1][30], b"Wrong Location") - self.assertEqual(rows[1][31], b"2021-05-25T02:27:19") + actual_headers = list(map(_str_if_bytes, rows[0])) + self.assertEqual(sorted(actual_headers), sorted(expected_column_headers)) + self.assertEqual(_str_if_bytes(rows[1][29]), "Rejected") + self.assertEqual(_str_if_bytes(rows[1][30]), "Wrong Location") + self.assertEqual(_str_if_bytes(rows[1][31]), "2021-05-25T02:27:19") # pylint: disable=invalid-name def test_zipped_csv_export_with_osm_data(self): From 7f47f200c2f45d6742fcd450be60b6479b62a6d3 Mon Sep 17 00:00:00 2001 From: Ukang'a Dickson Date: Fri, 5 Apr 2024 23:40:34 +0300 Subject: [PATCH 37/37] Flaky tests - string encoding --- .../libs/tests/utils/test_export_builder.py | 207 +++++++++--------- 1 file changed, 105 insertions(+), 102 deletions(-) diff --git a/onadata/libs/tests/utils/test_export_builder.py b/onadata/libs/tests/utils/test_export_builder.py index 7edf2723bd..f5c2d7ec8f 100644 --- a/onadata/libs/tests/utils/test_export_builder.py +++ b/onadata/libs/tests/utils/test_export_builder.py @@ -618,12 +618,12 @@ def test_zipped_sav_export_with_date_field(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"expense_date") - self.assertEqual(rows[1][0], b"2013-01-03") - self.assertEqual(rows[0][1], b"A.gdate") - self.assertEqual(rows[1][1], b"2017-06-13") - self.assertEqual(rows[0][5], b"@_submission_time") - self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + self.assertEqual(_str_if_bytes(rows[0][0]), "expense_date") + self.assertEqual(_str_if_bytes(rows[1][0]), "2013-01-03") + self.assertEqual(_str_if_bytes(rows[0][1]), "A.gdate") + self.assertEqual(_str_if_bytes(rows[1][1]), "2017-06-13") + self.assertEqual(_str_if_bytes(rows[0][5]), "@_submission_time") + self.assertEqual(_str_if_bytes(rows[1][5]), "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -670,24 +670,26 @@ def test_zipped_sav_export_dynamic_select_multiple(self): with SavReader(os.path.join(temp_dir, "exp.sav"), returnHeader=True) as reader: rows = list(reader) + rows[0] = list(map(_str_if_bytes, rows[0])) + rows[1] = list(map(_str_if_bytes, rows[1])) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"sex") - self.assertEqual(rows[1][0], b"male") - self.assertEqual(rows[0][1], b"text") - self.assertEqual(rows[1][1], b"his") - self.assertEqual(rows[0][2], b"favorite_brand") - self.assertEqual(rows[1][2], b"Generic") - self.assertEqual(rows[0][3], b"name") - self.assertEqual(rows[1][3], b"Davis") - self.assertEqual(rows[0][4], b"brand_known") - self.assertEqual(rows[1][4], b"his Generic a") - self.assertEqual(rows[0][5], b"brand_known.$text") + self.assertEqual(rows[0][0], "sex") + self.assertEqual(rows[1][0], "male") + self.assertEqual(rows[0][1], "text") + self.assertEqual(rows[1][1], "his") + self.assertEqual(rows[0][2], "favorite_brand") + self.assertEqual(rows[1][2], "Generic") + self.assertEqual(rows[0][3], "name") + self.assertEqual(rows[1][3], "Davis") + self.assertEqual(rows[0][4], "brand_known") + self.assertEqual(rows[1][4], "his Generic a") + self.assertEqual(rows[0][5], "brand_known.$text") self.assertEqual(rows[1][5], 1.0) - self.assertEqual(rows[0][6], b"brand_known.$favorite_brand") + self.assertEqual(rows[0][6], "brand_known.$favorite_brand") self.assertEqual(rows[1][6], 1.0) - self.assertEqual(rows[0][7], b"brand_known.a") + self.assertEqual(rows[0][7], "brand_known.a") self.assertEqual(rows[1][7], 1.0) - self.assertEqual(rows[0][8], b"brand_known.b") + self.assertEqual(rows[0][8], "brand_known.b") self.assertEqual(rows[1][8], 0.0) shutil.rmtree(temp_dir) @@ -764,16 +766,16 @@ def test_zipped_sav_export_with_numeric_select_one_field(self): self.assertTrue(len(rows) > 1) # expensed 1 - self.assertEqual(rows[0][0], b"expensed") - self.assertEqual(rows[1][0], 1) + self.assertEqual(_str_if_bytes(rows[0][0]), "expensed") + self.assertEqual(_str_if_bytes(rows[1][0]), 1) # A/q1 1 - self.assertEqual(rows[0][1], b"A.q1") + self.assertEqual(_str_if_bytes(rows[0][1]), "A.q1") self.assertEqual(rows[1][1], 1) # _submission_time is a date string - self.assertEqual(rows[0][5], b"@_submission_time") - self.assertEqual(rows[1][5], b"2016-11-21 03:43:43") + self.assertEqual(_str_if_bytes(rows[0][5]), "@_submission_time") + self.assertEqual(_str_if_bytes(rows[1][5]), "2016-11-21 03:43:43") # pylint: disable=invalid-name def test_zipped_sav_export_with_duplicate_field_different_groups(self): @@ -956,35 +958,35 @@ def test_zipped_sav_export_with_numeric_select_multiple_field(self): rows = list(reader) self.assertTrue(len(rows) > 1) - self.assertEqual(rows[0][0], b"expensed") - self.assertEqual(rows[1][0], b"1") + self.assertEqual(_str_if_bytes(rows[0][0]), "expensed") + self.assertEqual(_str_if_bytes(rows[1][0]), "1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][1], b"expensed.1") + self.assertEqual(_str_if_bytes(rows[0][1]), "expensed.1") self.assertEqual(rows[1][1], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][2], b"expensed.0") + self.assertEqual(_str_if_bytes(rows[0][2]), "expensed.0") self.assertEqual(rows[1][2], 0) - self.assertEqual(rows[0][3], b"A.q1") - self.assertEqual(rows[1][3], b"1") + self.assertEqual(_str_if_bytes(rows[0][3]), "A.q1") + self.assertEqual(_str_if_bytes(rows[1][3]), "1") # ensure you get a numeric value for multiple select with choice # filters - self.assertEqual(rows[0][6], b"A.q2") - self.assertEqual(rows[1][6], b"1") + self.assertEqual(_str_if_bytes(rows[0][6]), "A.q2") + self.assertEqual(_str_if_bytes(rows[1][6]), "1") # expensed.1 is selected hence True, 1.00 or 1 in SPSS - self.assertEqual(rows[0][4], b"A.q1.1") + self.assertEqual(_str_if_bytes(rows[0][4]), "A.q1.1") self.assertEqual(rows[1][4], 1) # expensed.0 is not selected hence False, .00 or 0 in SPSS - self.assertEqual(rows[0][5], b"A.q1.0") + self.assertEqual(_str_if_bytes(rows[0][5]), "A.q1.0") self.assertEqual(rows[1][5], 0) - self.assertEqual(rows[0][12], b"@_submission_time") - self.assertEqual(rows[1][12], b"2016-11-21 03:43:43") + self.assertEqual(_str_if_bytes(rows[0][12]), "@_submission_time") + self.assertEqual(_str_if_bytes(rows[1][12]), "2016-11-21 03:43:43") shutil.rmtree(temp_dir) @@ -1170,10 +1172,10 @@ def test_zipped_sav_export_with_duplicate_column_name(self): rows = list(reader) # Check that columns are present - self.assertIn(b"Sport", rows[0]) + self.assertIn("Sport", _str_if_bytes(rows[0])) # Check for sport in first 5 characters # because rows contains 'sport@d4b6' - self.assertIn(b"sport", [x[0:5] for x in rows[0]]) + self.assertIn("sport", list(map(_str_if_bytes, [x[0:5] for x in rows[0]]))) # pylint: disable=invalid-name def test_xlsx_export_works_with_unicode(self): @@ -3044,46 +3046,45 @@ def test_zipped_sav_export_with_osm_data(self): with SavReader(os.path.join(temp_dir, "osm.sav"), returnHeader=True) as reader: rows = list(reader) expected_column_headers = [ - x.encode("8") - for x in [ - "photo", - "osm_road", - "osm_building", - "fav_color", - "form_completed", - "meta.instanceID", - "@_id", - "@_uuid", - "@_submission_time", - "@_index", - "@_parent_table_name", - "@_parent_index", - "@_tags", - "@_notes", - "@_version", - "@_duration", - "@_submitted_by", - "osm_road_ctr_lat", - "osm_road_ctr_lon", - "osm_road_highway", - "osm_road_lanes", - "osm_road_name", - "osm_road_way_id", - "osm_building_building", - "osm_building_building_levels", - "osm_building_ctr_lat", - "osm_building_ctr_lon", - "osm_building_name", - "osm_building_way_id", - ] + "photo", + "osm_road", + "osm_building", + "fav_color", + "form_completed", + "meta.instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", + "osm_road_ctr_lat", + "osm_road_ctr_lon", + "osm_road_highway", + "osm_road_lanes", + "osm_road_name", + "osm_road_way_id", + "osm_building_building", + "osm_building_building_levels", + "osm_building_ctr_lat", + "osm_building_ctr_lon", + "osm_building_name", + "osm_building_way_id", ] + rows[0] = list(map(_str_if_bytes, rows[0])) + rows[1] = list(map(_str_if_bytes, rows[1])) self.assertEqual(sorted(rows[0]), sorted(expected_column_headers)) - self.assertEqual(rows[1][0], b"1424308569120.jpg") - self.assertEqual(rows[1][1], b"OSMWay234134797.osm") - self.assertEqual(rows[1][2], b"23.708174238006087") - self.assertEqual(rows[1][4], b"tertiary") - self.assertEqual(rows[1][6], b"Patuatuli Road") - self.assertEqual(rows[1][13], b"kol") + self.assertEqual(rows[1][0], "1424308569120.jpg") + self.assertEqual(rows[1][1], "OSMWay234134797.osm") + self.assertEqual(rows[1][2], "23.708174238006087") + self.assertEqual(rows[1][4], "tertiary") + self.assertEqual(rows[1][6], "Patuatuli Road") + self.assertEqual(rows[1][13], "kol") def test_show_choice_labels(self): """ @@ -3610,36 +3611,36 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): expected_data = [ [ - b"gps", - b"@_gps_latitude", - b"@_gps_longitude", - b"@_gps_altitude", - b"@_gps_precision", - b"gps@52a9", - b"@_gps_latitude_52a9", - b"@_gps_longitude_52a9", - b"@_gps_altitude_52a9", - b"@_gps_precision_52a9", - b"instanceID", - b"@_id", - b"@_uuid", - b"@_submission_time", - b"@_index", - b"@_parent_table_name", - b"@_parent_index", - b"@_tags", - b"@_notes", - b"@_version", - b"@_duration", - b"@_submitted_by", + "gps", + "@_gps_latitude", + "@_gps_longitude", + "@_gps_altitude", + "@_gps_precision", + "gps@52a9", + "@_gps_latitude_52a9", + "@_gps_longitude_52a9", + "@_gps_altitude_52a9", + "@_gps_precision_52a9", + "instanceID", + "@_id", + "@_uuid", + "@_submission_time", + "@_index", + "@_parent_table_name", + "@_parent_index", + "@_tags", + "@_notes", + "@_version", + "@_duration", + "@_submitted_by", ], [ - b"4.0 36.1 5000 20", + "4.0 36.1 5000 20", 4.0, 36.1, 5000.0, 20.0, - b"1.0 36.1 2000 20", + "1.0 36.1 2000 20", 1.0, 36.1, 2000.0, @@ -3647,7 +3648,7 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): "", None, "", - b"2016-11-21 03:42:43", + "2016-11-21 03:42:43", 1.0, "", -1.0, @@ -3663,5 +3664,7 @@ def test_sav_export_with_duplicate_metadata(self, mock_uuid): ) as reader: rows = list(reader) self.assertEqual(len(rows), 2) + rows[0] = list(map(_str_if_bytes, rows[0])) + rows[1] = list(map(_str_if_bytes, rows[1])) self.assertEqual(expected_data, rows) shutil.rmtree(temp_dir)