From 9a0c712531bd412507f3baaaf85d143439feece3 Mon Sep 17 00:00:00 2001 From: Oleh Kuchuk Date: Sat, 20 Apr 2019 17:10:08 +0300 Subject: [PATCH 1/2] Initialized asyncpg as an option for postgres engine --- create_aio_app/main.py | 2 +- create_aio_app/template/cookiecutter.json | 2 +- .../{{cookiecutter.project_name}}/Makefile | 6 ++-- .../{{cookiecutter.project_name}}/README.md | 4 +-- .../config/api.dev.yml | 2 +- .../config/api.prod.yml | 2 +- .../config/api.test.yml | 2 +- .../{{cookiecutter.project_name}}/conftest.py | 28 ++++++++++++------- .../docker-compose.yml | 6 ++-- .../requirements/production.txt | 8 ++++-- .../{{cookiecutter.project_name}}/.flake8 | 2 +- .../{{cookiecutter.project_name}}/app.py | 15 +++++++--- .../{{cookiecutter.project_name}}/types.py | 4 +-- .../users/db_utils.py | 27 ++++++++++++++---- .../users/tables.py | 7 +++-- .../utils/common.py | 2 +- .../utils/wait_script.py | 2 +- create_aio_app/utils/config.py | 6 ++++ 18 files changed, 84 insertions(+), 43 deletions(-) diff --git a/create_aio_app/main.py b/create_aio_app/main.py index f001bcf2..51530670 100644 --- a/create_aio_app/main.py +++ b/create_aio_app/main.py @@ -26,7 +26,7 @@ def main(): 'no_input': True, 'extra_context': { 'project_name': args.get('name'), - 'use_postgres': 'n' if args.get('without_postgres') else 'y', + 'use_postgres': 'n' if args.get('without_postgres') else args['postgres_engine'], 'use_redis': 'y' if args.get('redis') else 'n', }, } diff --git a/create_aio_app/template/cookiecutter.json b/create_aio_app/template/cookiecutter.json index 907f873b..7594c6f5 100644 --- a/create_aio_app/template/cookiecutter.json +++ b/create_aio_app/template/cookiecutter.json @@ -1,5 +1,5 @@ { "project_name": "app", - "use_postgres": "y", + "use_postgres": ["aiopg", "asyncpg", "n"], "use_redis": "n" } diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/Makefile b/create_aio_app/template/{{cookiecutter.project_name}}/Makefile index c807c7e7..b4dd9a63 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/Makefile +++ b/create_aio_app/template/{{cookiecutter.project_name}}/Makefile @@ -44,7 +44,7 @@ test: lint @docker-compose up test @docker-compose stop test -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} # Database psql: @@ -61,11 +61,11 @@ migrate: run_production: @docker-compose -f docker-compose.yml -f docker-compose.production.yml up -adev: {%- if cookiecutter.use_postgres == 'y' or cookiecutter.use_redis == 'y' %} wait_resources {% endif %} +adev: {%- if cookiecutter.use_postgres != 'n' or cookiecutter.use_redis == 'y' %} wait_resources {% endif %} adev runserver ./{{ cookiecutter.project_name }}/__main__.py -p 8080 -{%- if cookiecutter.use_postgres == 'y' or cookiecutter.use_redis == 'y' %} +{%- if cookiecutter.use_postgres != 'n' or cookiecutter.use_redis == 'y' %} wait_resources: python3 -m {{ cookiecutter.project_name }}.utils.wait_script {%- endif %} diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/README.md b/create_aio_app/template/{{cookiecutter.project_name}}/README.md index 1c3288c0..df81a14d 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/README.md +++ b/create_aio_app/template/{{cookiecutter.project_name}}/README.md @@ -17,7 +17,7 @@ ___ - docker-compose - aio devtools - aiohttp debug toolbar -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} - postgres - alembic - aiopg @@ -85,7 +85,7 @@ make mypy The all settings connected with a `mypy` you can customize in `mypy.ini`. ___ -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} ### Testing ``` diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.dev.yml b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.dev.yml index d330e056..06b5f61d 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.dev.yml +++ b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.dev.yml @@ -2,7 +2,7 @@ app: host: 0.0.0.0 port: 8000 -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} # postgres postgres: diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.prod.yml b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.prod.yml index e08e4895..dee47132 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.prod.yml +++ b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.prod.yml @@ -2,7 +2,7 @@ app: host: 0.0.0.0 port: 8080 -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} # postgres postgres: diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.test.yml b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.test.yml index a227eeb7..6f6a5f76 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/config/api.test.yml +++ b/create_aio_app/template/{{cookiecutter.project_name}}/config/api.test.yml @@ -2,7 +2,7 @@ app: host: 0.0.0.0 port: 9000 -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} # postgres postgres: diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/conftest.py b/create_aio_app/template/{{cookiecutter.project_name}}/conftest.py index 1306ce79..80599647 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/conftest.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/conftest.py @@ -1,14 +1,14 @@ import pytest -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres == 'aiopg' %} from sqlalchemy import create_engine import aiopg.sa {%- endif %} -from {{ cookiecutter.project_name }}.utils.common import PATH, get_config -from {{ cookiecutter.project_name }}.app import init_app -{%- if cookiecutter.use_postgres == 'y' %} -from {{ cookiecutter.project_name }}.migrations import metadata -from {{ cookiecutter.project_name }}.users.tables import users +from {{cookiecutter.project_name}}.utils.common import PATH, get_config +from {{cookiecutter.project_name}}.app import init_app +{%- if cookiecutter.use_postgres != 'n' %} +from {{cookiecutter.project_name}}.migrations import metadata +from {{cookiecutter.project_name}}.users.tables import users {%- endif %} @@ -20,10 +20,9 @@ test_config = get_config(['-c', TEST_CONFIG_PATH.as_posix()]) -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} # helpers - def get_db_url(config: dict) -> str: ''' Generate a url for db connection from the config. @@ -58,7 +57,7 @@ def init_sample_data(engine) -> None: 'username': f'test#{idx}', 'email': f'test#{idx}', 'password': f'{idx}'} for idx in range(10) - ]) + ]) conn.execute(query) @@ -138,11 +137,20 @@ async def sa_engine(loop): ''' return await aiopg.sa.create_engine(**test_config['postgres']) + + +@pytest.fixture +async def postgres_engine(loop): + {%- if cookiecutter.use_postgres == 'aiopg' %} + import asyncpg + return await asyncpg.connect(**test_config['postgres']) + {%- endif %} + {%- endif %} @pytest.fixture -async def client(aiohttp_client{% if cookiecutter.use_postgres == 'y' %}, tables{% endif %}): +async def client(aiohttp_client {% if cookiecutter.use_postgres != 'n' %} , tables{% endif %}): ''' The fixture for the initialize client. ''' diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/docker-compose.yml b/create_aio_app/template/{{cookiecutter.project_name}}/docker-compose.yml index 55ee2245..c99c5d03 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/docker-compose.yml +++ b/create_aio_app/template/{{cookiecutter.project_name}}/docker-compose.yml @@ -17,11 +17,11 @@ services: restart: always volumes: - .:/app:delegated - {%- if cookiecutter.use_postgres == 'y' and cookiecutter.use_redis == 'y' %} + {%- if cookiecutter.use_postgres != 'n' and cookiecutter.use_redis == 'y' %} depends_on: - postgres - redis - {%- elif cookiecutter.use_postgres == 'y' %} + {%- elif cookiecutter.use_postgres != 'n' %} depends_on: - postgres {%- elif cookiecutter.use_redis == 'y' %} @@ -32,7 +32,7 @@ services: - 8080:8080 - 8081:8081 command: make adev - {%- if cookiecutter.use_postgres == 'y' %} + {%- if cookiecutter.use_postgres != 'n' %} postgres: image: postgres:10 diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/requirements/production.txt b/create_aio_app/template/{{cookiecutter.project_name}}/requirements/production.txt index 1414fab5..8f268df8 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/requirements/production.txt +++ b/create_aio_app/template/{{cookiecutter.project_name}}/requirements/production.txt @@ -2,11 +2,15 @@ aiohttp==3.5.4 aiohttp_jinja2==1.1.0 trafaret_config==2.0.2 markdown2==2.3.7 -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres == 'aiopg' %} aiopg[sa]==0.16.0 psycopg2-binary==2.7.7 +{%- elif cookiecutter.use_postgres == 'asyncpg' %} +asyncpg +{%- endif %} +{%- if cookiecutter.use_postgres != 'n' %} alembic==1.0.7 {%- endif %} {%- if cookiecutter.use_redis == 'y' %} aioredis==1.2.0 -{%- endif %} \ No newline at end of file +{%- endif %} diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/.flake8 b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/.flake8 index 35e5423e..7bbdc9ae 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/.flake8 +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/.flake8 @@ -1,3 +1,3 @@ [flake8] max-line-length = 79 -exclude = env/* {% if cookiecutter.use_postgres == 'y' %}migrations/*{% endif %} +exclude = env/* {% if cookiecutter.use_postgres != 'n' %}migrations/*{% endif %} diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/app.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/app.py index b41472d0..5aad33d5 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/app.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/app.py @@ -6,7 +6,10 @@ {%- endif %} import aiohttp_jinja2 -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres == 'asyncpg' %} +import asyncpg +{%- endif %} +{%- if cookiecutter.use_postgres == 'aiopg' %} import aiopg.sa {%- endif %} from aiohttp import web @@ -30,7 +33,7 @@ def init_jinja2(app: web.Application) -> None: app, loader=jinja2.FileSystemLoader(str(path / 'templates')) ) -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres != 'n' %} async def database(app: web.Application) -> None: @@ -40,7 +43,11 @@ async def database(app: web.Application) -> None: ''' config = app['config']['postgres'] + {%- if cookiecutter.use_postgres == 'aiopg' %} engine = await aiopg.sa.create_engine(**config) + {%- elif cookiecutter.use_postgres == 'asyncpg' %} + engine = await asyncpg.connect(**config) + {%- endif %} app['db'] = engine yield @@ -86,13 +93,13 @@ def init_app(config: Optional[List[str]] = None) -> web.Application: init_jinja2(app) init_config(app, config=config) init_routes(app) - {%- if cookiecutter.use_postgres == 'y' and cookiecutter.use_redis == 'y' %} + {%- if cookiecutter.use_postgres != 'n' and cookiecutter.use_redis == 'y' %} app.cleanup_ctx.extend([ redis, database, ]) - {%- elif cookiecutter.use_postgres == 'y' %} + {%- elif cookiecutter.use_postgres != 'n' %} app.cleanup_ctx.extend([ database, diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/types.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/types.py index c39fd779..2deb56b8 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/types.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/types.py @@ -1,8 +1,8 @@ -{%- if cookiecutter.use_postgres == 'y' %} +{%- if cookiecutter.use_postgres == 'aiopg' %} from typing import List from aiopg.sa.result import RowProxy RowsProxy = List[RowProxy] -{% endif %} \ No newline at end of file +{%- endif %} diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/db_utils.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/db_utils.py index 0df5ca46..dc8878bf 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/db_utils.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/db_utils.py @@ -1,17 +1,32 @@ +{%- if cookiecutter.use_postgres == 'asyncpg' %} +import asyncpg +{%- endif %} +{%- if cookiecutter.use_postgres == 'aiopg' %} from aiopg.sa import SAConnection from aiopg.sa.result import RowProxy +{%- endif %} +from {{cookiecutter.project_name}}.users.tables import users{%- if cookiecutter.use_postgres == 'asyncpg' %}, dialect {%- endif %} -from {{ cookiecutter.project_name }}.users.tables import users +__all__ = ['select_user_by_id', 'select_user_by_id_query'] -__all__ = ['select_user_by_id', ] - - -async def select_user_by_id(conn: SAConnection, key: int) -> RowProxy: +def select_user_by_id_query(key): query = users\ .select()\ .where(users.c.id == key)\ .order_by(users.c.id) - cursor = await conn.execute(query) + return query + + +{%- if cookiecutter.use_postgres == 'asyncpg' %} +async def select_user_by_id(conn, key): + query = str(select_user_by_id_query(key).compile(dialect=dialect)) + result = await conn.fetchrow(query) + return result +{%- endif %} +{%- if cookiecutter.use_postgres == 'aiopg' %} +async def select_user_by_id(conn: SAConnection, key: int) -> RowProxy: + cursor = await conn.execute(select_user_by_id_query(key)) return await cursor.fetchone() +{%- endif %} diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/tables.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/tables.py index 42080833..f9edc814 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/tables.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/users/tables.py @@ -1,15 +1,16 @@ import sqlalchemy as sa from sqlalchemy.dialects import postgresql -from {{ cookiecutter.project_name }}.migrations import metadata -from {{ cookiecutter.project_name }}.users.enums import UserGender +from {{cookiecutter.project_name}}.migrations import metadata +from {{cookiecutter.project_name}}.users.enums import UserGender -__all__ = ['users', 'gender_enum', ] +__all__ = ['users', 'gender_enum', 'dialect', ] gender_enum = postgresql.ENUM(UserGender) +dialect = postgresql.dialect() users = sa.Table( 'users', metadata, diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/common.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/common.py index ae6a1dd8..f050ad46 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/common.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/common.py @@ -19,7 +19,7 @@ 'host': trafaret.String(), 'port': trafaret.Int(), }), - {%- if cookiecutter.use_postgres == 'y' %} + {%- if cookiecutter.use_postgres != 'n' %} trafaret.Key('postgres'): trafaret.Dict({ 'host': trafaret.String(), diff --git a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/wait_script.py b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/wait_script.py index 982e9d99..6451506c 100644 --- a/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/wait_script.py +++ b/create_aio_app/template/{{cookiecutter.project_name}}/{{cookiecutter.project_name}}/utils/wait_script.py @@ -6,7 +6,7 @@ if __name__ == '__main__': config = get_config(['-c', DEFAULT_CONFIG_PATH.as_posix()]) - {%- if cookiecutter.use_postgres == 'y' %} + {%- if cookiecutter.use_postgres != 'n' %} postgres = config['postgres'] while True: try: diff --git a/create_aio_app/utils/config.py b/create_aio_app/utils/config.py index b2679ec9..13996e21 100644 --- a/create_aio_app/utils/config.py +++ b/create_aio_app/utils/config.py @@ -46,5 +46,11 @@ def parse_arguments() -> dict: action='store_true', help='generate project without postgres settings' ) + parser.add_argument( + '--postgres_engine', + default='aiopg', + choices=['aiopg', 'asyncpg'], + help='choose which postgres engine to use', + ) return parser.parse_args().__dict__ From 79d2c28c5107bfbf606e13f83a0a4d8fa5a92f9b Mon Sep 17 00:00:00 2001 From: Oleh Kuchuk Date: Sat, 20 Apr 2019 17:27:04 +0300 Subject: [PATCH 2/2] Fix flake8 --- create_aio_app/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_aio_app/main.py b/create_aio_app/main.py index 51530670..72c17842 100644 --- a/create_aio_app/main.py +++ b/create_aio_app/main.py @@ -26,7 +26,7 @@ def main(): 'no_input': True, 'extra_context': { 'project_name': args.get('name'), - 'use_postgres': 'n' if args.get('without_postgres') else args['postgres_engine'], + 'use_postgres': 'n' if args.get('without_postgres') else args['postgres_engine'], # noqa 'use_redis': 'y' if args.get('redis') else 'n', }, }