Merge branch 'master' into production

.babelrc

@@ -1,3 +1,5 @@
 {
-  "presets": ["es2015", "stage-2"],
+  "presets": [
+    "@babel/preset-env"
+  ]
 }

.dockerignore

@@ -1,7 +1,6 @@
 .git
 
 # Docker configuration
-/docker
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -1,4 +1,4 @@
-FROM metabrainz/python:3.7
+FROM metabrainz/python:3.8
 
 ARG DEPLOY_ENV
 
@@ -18,7 +18,7 @@ RUN apt-get update \
 
 # PostgreSQL client
 RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8
-ENV PG_MAJOR 9.5
+ENV PG_MAJOR 12
 RUN echo 'deb http://apt.postgresql.org/pub/repos/apt/ jessie-pgdg main' $PG_MAJOR > /etc/apt/sources.list.d/pgdg.list
 RUN apt-get update \
     && apt-get install -y postgresql-client-$PG_MAJOR \
@@ -27,27 +27,27 @@ RUN apt-get update \
 ENV PGPASSWORD "critiquebrainz"
 
 # Node
-RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -
-RUN apt-get install -y nodejs
+RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - \
+   && apt-get install -y nodejs \
+   && rm -rf /var/lib/apt/lists/*
 
-RUN pip install uWSGI==2.0.13.1
+RUN pip install --no-cache-dir uWSGI==2.0.18
 
 RUN mkdir /code
 WORKDIR /code
 
 # Python dependencies
 COPY ./requirements.txt /code/
-RUN pip install -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Node dependencies
-COPY ./package.json /code/
-COPY ./npm-shrinkwrap.json /code/
+COPY ./package.json ./package-lock.json /code/
 RUN npm install
 
 COPY . /code/
 
 # Build static files
-RUN ./node_modules/.bin/gulp
+RUN npm run build
 
 # Compile translations
 RUN pybabel compile -d critiquebrainz/frontend/translations

Dockerfile.webpack

@@ -0,0 +1,7 @@
+FROM node:12
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY package.json package-lock.json webpack.config.js /code/
+RUN npm install

README.md

@@ -4,12 +4,11 @@ CritiqueBrainz is a repository for Creative Commons licensed reviews.
 
 ## Documentation
 
-**Documentation is available at https://critiquebrainz.readthedocs.org/.**
+Documentation is available at https://critiquebrainz.readthedocs.io/.
 
 ### Building documentation
 
-In case you want to build documentation yourself follow the following steps
-*(make sure that you use Python 3)*:
+If you want to build documentation yourself, follow these steps using python 3
 
     $ cd ./docs
     $ pip3 install -r requirements.txt

admin/schema_changes/16.sql

@@ -0,0 +1 @@
+ALTER TYPE entity_types ADD VALUE 'artist' AFTER 'place';

admin/schema_changes/17.sql

@@ -0,0 +1 @@
+ALTER TYPE entity_types ADD VALUE 'label' AFTER 'artist';

admin/schema_changes/19.sql

@@ -0,0 +1 @@
+ALTER TYPE entity_types ADD VALUE 'work' AFTER 'recording';

admin/sql/create_types.sql

@@ -8,5 +8,8 @@ CREATE TYPE action_types AS ENUM (
 CREATE TYPE entity_types AS ENUM (
     'release_group',
     'event',
-    'place'
+    'place',
+    'work',
+    'artist',
+    'label'
 );

critiquebrainz/data/dump_manager.py

@@ -1,24 +1,25 @@
-from time import gmtime, strftime
-from datetime import datetime
-import subprocess
-import tempfile
-import tarfile
-import shutil
 import errno
-import sys
 import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import tempfile
+from datetime import datetime
+from time import gmtime, strftime
+
+import click
+import sqlalchemy
 from flask import current_app, jsonify
 from flask.json import JSONEncoder
-import sqlalchemy
-import click
+from psycopg2.sql import SQL, Identifier
+
+from critiquebrainz import db
 from critiquebrainz.data.utils import create_path, remove_old_archives, slugify, explode_db_uri, with_request_context
 from critiquebrainz.db import license as db_license, review as db_review
-from critiquebrainz import db
-
 
 cli = click.Group()
 
-
 _TABLES = {
     "review": (
         "id",
@@ -66,6 +67,12 @@
         "rating",
         "count",
     ),
+    "vote": (
+        "user_id",
+        "revision_id",
+        "vote",
+        "rated_at",
+    ),
 }
 
 
@@ -104,7 +111,7 @@ def full_db(location, rotate=False):
     # More info about it is available at http://www.postgresql.org/docs/9.3/static/app-pgdump.html
     dump_file = os.path.join(location, FILE_PREFIX + strftime("%Y%m%d-%H%M%S", gmtime()))
     print('pg_dump -h "%s" -p "%s" -U "%s" -d "%s" -Ft > "%s.tar"' %
-          (db_hostname, db_port, db_username, db_name, dump_file),)
+          (db_hostname, db_port, db_username, db_name, dump_file), )
     result = subprocess.call(
         'pg_dump -h "%s" -p "%s" -U "%s" -d "%s" -Ft > "%s.tar"' %
         (db_hostname, db_port, db_username, db_name, dump_file),
@@ -159,7 +166,8 @@ def json(location, rotate=False):
                     entity = str(entity)
                     # Creating directory structure and dumping reviews
                     dir_part = os.path.join(entity[0:1], entity[0:2])
-                    reviews = db_review.get_reviews_list(connection, entity_id=entity, license_id=license["id"], limit=None)[0]
+                    reviews = db_review.get_reviews_list(connection, entity_id=entity,
+                                                         license_id=license["id"], limit=None)[0]
                     if reviews:
                         rg_dir = '%s/%s' % (license_dir, dir_part)
                         create_path(rg_dir)
@@ -174,7 +182,8 @@ def json(location, rotate=False):
                 tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", safe_name + ".txt"),
                         arcname='COPYING')
 
-                print(" + %s/critiquebrainz-%s-%s-json.tar.bz2" % (location, datetime.today().strftime('%Y%m%d'), safe_name))
+                print(" + %s/critiquebrainz-%s-%s-json.tar.bz2" % (
+                    location, datetime.today().strftime('%Y%m%d'), safe_name))
 
                 shutil.rmtree(temp_dir)  # Cleanup
 
@@ -307,7 +316,8 @@ def create_base_archive(connection, *, location, meta_files_dir=None):
 
         # Including additional information about this archive
         # Copying the most restrictive license there (CC BY-NC-SA 3.0)
-        tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", "cc-by-nc-sa-30.txt"), arcname='COPYING')
+        tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", "cc-by-nc-sa-30.txt"),
+                arcname='COPYING')
         # Copy meta files
         if not meta_files_dir:
             prepare_meta_files(temp_dir)
@@ -364,6 +374,23 @@ def create_reviews_archive(connection, *, location, meta_files_dir=None, license
         license_where_clause=license_where_clause,
     )
 
+    VOTE_SQL = """(
+        SELECT {columns}
+          FROM vote
+          JOIN ( SELECT revision.id
+                   FROM revision
+                   JOIN review
+                     ON review.id = revision.review_id
+                  WHERE review.is_hidden = false
+                    AND review.is_draft = false
+                        {license_where_clause}
+                ) AS rev
+            ON vote.revision_id = rev.id
+    )""".format(
+        columns=', '.join(['vote.' + column for column in _TABLES['vote']]),
+        license_where_clause=license_where_clause,
+    )
+
     with tarfile.open(os.path.join(location, archive_name), "w:bz2") as tar:
         # Dumping tables
         temp_dir = tempfile.mkdtemp()
@@ -380,6 +407,10 @@ def create_reviews_archive(connection, *, location, meta_files_dir=None, license
 
             with open(os.path.join(reviews_tables_dir, 'avg_rating'), 'w') as f:
                 cursor.copy_to(f, "(SELECT {columns} FROM avg_rating)".format(columns=", ".join(_TABLES["avg_rating"])))
+
+            with open(os.path.join(reviews_tables_dir, 'vote'), 'w') as f:
+                cursor.copy_to(f, VOTE_SQL)
+
         except Exception as e:
             print("Error {} occurred while copying tables during the creation of the reviews archive!".format(e))
             raise
@@ -389,7 +420,8 @@ def create_reviews_archive(connection, *, location, meta_files_dir=None, license
             tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", "cc-by-nc-sa-30.txt"),
                     arcname='COPYING')
         else:
-            tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", safe_name + ".txt"), arcname='COPYING')
+            tar.add(os.path.join(os.path.dirname(os.path.realpath(__file__)), "licenses", safe_name + ".txt"),
+                    arcname='COPYING')
 
         if not meta_files_dir:
             prepare_meta_files(temp_dir)
@@ -442,13 +474,16 @@ def importer(archive):
         import_data(os.path.join(temp_dir, 'cbdump', 'review'), 'review')
         import_data(os.path.join(temp_dir, 'cbdump', 'revision'), 'revision')
         import_data(os.path.join(temp_dir, 'cbdump', 'avg_rating'), 'avg_rating')
+        import_data(os.path.join(temp_dir, 'cbdump', 'vote'), 'vote')
+
+        # Reset sequence values after importing dump
+        reset_sequence(['revision'])
 
         shutil.rmtree(temp_dir)  # Cleanup
         print("Done!")
 
 
 def import_data(file_name, table_name, columns=None):
-
     connection = db.engine.raw_connection()
     try:
         cursor = connection.cursor()
@@ -475,6 +510,19 @@ def import_data(file_name, table_name, columns=None):
         connection.close()
 
 
+def reset_sequence(table_names):
+    connection = db.engine.raw_connection()
+    try:
+        cursor = connection.cursor()
+        for table_name in table_names:
+            cursor.execute(
+                SQL("SELECT setval(pg_get_serial_sequence(%s, 'id'), coalesce(max(id),0) + 1, false) FROM {};")
+                .format(Identifier(table_name)), (table_name,))
+            connection.commit()
+    finally:
+        connection.close()
+
+
 class DumpJSONEncoder(JSONEncoder):
     """Custom JSON encoder for database dumps."""
 

critiquebrainz/data/fixtures.py

@@ -37,4 +37,4 @@ class LicenseData:
 
 
 # Include all objects into this tuple.
-all_data = (LicenseData, )
+all_data = (LicenseData,)

critiquebrainz/data/mixins.py

@@ -4,6 +4,7 @@
 
 class AdminMixin(UserMixin):
     """Allows a method to check if the current user is admin."""
+
     def is_admin(self):
         return self.musicbrainz_username in current_app.config['ADMINS']
 

critiquebrainz/data/testing.py

@@ -1,7 +1,9 @@
 import os
+
 from flask_testing import TestCase
-from critiquebrainz.frontend import create_app
+
 from critiquebrainz.data.utils import create_all, drop_tables, drop_types
+from critiquebrainz.frontend import create_app
 
 
 class DataTestCase(TestCase):

critiquebrainz/data/utils.py

@@ -1,11 +1,12 @@
-import urllib.parse
-import unicodedata
-import shutil
 import errno
-import sys
 import os
 import re
+import shutil
+import sys
+import unicodedata
+import urllib.parse
 from functools import wraps
+
 from critiquebrainz import db
 from critiquebrainz import frontend
 
@@ -102,16 +103,19 @@ def with_request_context(f):
     def decorated(*args, **kwargs):
         with frontend.create_app().test_request_context():
             return f(*args, **kwargs)
+
     return decorated
 
 
 def with_test_request_context(f):
     """Decorator for providing request context for application using test_config.py."""
+
     @wraps(f)
     def decorated(*args, **kwargs):
         with frontend.create_app(
                 config_path=os.path.join(
                     os.path.dirname(os.path.realpath(__file__)),
                     '..', 'test_config.py')).test_request_context():
             return f(*args, **kwargs)
+
     return decorated

critiquebrainz/data/utils_test.py

@@ -1,5 +1,5 @@
-from critiquebrainz.data.testing import DataTestCase
 from critiquebrainz.data import utils
+from critiquebrainz.data.testing import DataTestCase
 
 
 class DataUtilsTestCase(DataTestCase):

critiquebrainz/db/avg_rating.py

@@ -1,4 +1,5 @@
 import sqlalchemy
+
 from critiquebrainz import db
 from critiquebrainz.db import exceptions as db_exceptions
 

critiquebrainz/db/avg_rating_test.py

@@ -1,10 +1,10 @@
-from critiquebrainz.data.testing import DataTestCase
-import critiquebrainz.db.users as db_users
-from critiquebrainz.db.user import User
-import critiquebrainz.db.review as db_review
 import critiquebrainz.db.avg_rating as db_avg_rating
 import critiquebrainz.db.exceptions as db_exceptions
 import critiquebrainz.db.license as db_license
+import critiquebrainz.db.review as db_review
+import critiquebrainz.db.users as db_users
+from critiquebrainz.data.testing import DataTestCase
+from critiquebrainz.db.user import User
 
 
 class AvgRatingTestCase(DataTestCase):