Merge branch 'master' into edly/forumv2

README.rst

@@ -100,6 +100,8 @@ Language Packages:
   - ``pip install -r requirements/edx/base.txt`` (production)
   - ``pip install -r requirements/edx/dev.txt`` (development)
 
+  Some Python packages have system dependencies. For example, installing these packages on Debian or Ubuntu will require first running ``sudo apt install python3-dev default-libmysqlclient-dev build-essential pkg-config`` to satisfy the requirements of the ``mysqlclient`` Python package.
+
 Build Steps
 -----------
 

cms/djangoapps/contentstore/asset_storage_handlers.py

@@ -25,6 +25,7 @@
 from common.djangoapps.util.json_request import JsonResponse
 from openedx.core.djangoapps.contentserver.caching import del_cached_content
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
+from openedx_filters.course_authoring.filters import LMSPageURLRequested
 from xmodule.contentstore.content import StaticContent  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.contentstore.django import contentstore  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.exceptions import NotFoundError  # lint-amnesty, pylint: disable=wrong-import-order
@@ -714,7 +715,15 @@ def get_asset_json(display_name, content_type, date, location, thumbnail_locatio
     Helper method for formatting the asset information to send to client.
     '''
     asset_url = StaticContent.serialize_asset_key_with_slash(location)
-    external_url = urljoin(configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL), asset_url)
+
+    ## .. filter_implemented_name: LMSPageURLRequested
+    ## .. filter_type: org.openedx.course_authoring.lms.page.url.requested.v1
+    lms_root, _ = LMSPageURLRequested.run_filter(
+        url=configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL),
+        org=location.org,
+    )
+
+    external_url = urljoin(lms_root, asset_url)
     portable_url = StaticContent.get_static_path_from_location(location)
     usage_locations = [] if usage is None else usage
     return {

cms/djangoapps/contentstore/helpers.py

@@ -190,6 +190,8 @@ def xblock_type_display_name(xblock, default_display_name=None):
         # description like "Multiple Choice Problem", but that won't work if our 'block' argument is just the block_type
         # string ("problem").
         return _('Problem')
+    elif category == 'library_v2':
+        return _('Library Content')
     component_class = XBlock.load_class(category)
     if hasattr(component_class, 'display_name') and component_class.display_name.default:
         return _(component_class.display_name.default)  # lint-amnesty, pylint: disable=translation-of-non-string
@@ -267,7 +269,6 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request) ->
     empty, and (2) a summary of changes made to static files in the destination
     course.
     """
-
     from cms.djangoapps.contentstore.views.preview import _load_preview_block
 
     if not content_staging_api:
@@ -324,6 +325,8 @@ def _import_xml_node_to_parent(
     Given an XML node representing a serialized XBlock (OLX), import it into modulestore 'store' as a child of the
     specified parent block. Recursively copy children as needed.
     """
+    # pylint: disable=too-many-statements
+
     runtime = parent_xblock.runtime
     parent_key = parent_xblock.scope_ids.usage_id
     block_type = node.tag
@@ -429,7 +432,14 @@ def _import_xml_node_to_parent(
             )
 
     # Copy content tags to the new xblock
-    if copied_from_block and tags:
+    if new_xblock.upstream:
+        # If this block is synced from an upstream (e.g. library content),
+        # copy the tags from the upstream as ready-only
+        content_tagging_api.copy_tags_as_read_only(
+            new_xblock.upstream,
+            new_xblock.location,
+        )
+    elif copied_from_block and tags:
         object_tags = tags.get(str(copied_from_block))
         if object_tags:
             content_tagging_api.set_all_object_tags(

cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py

@@ -89,6 +89,7 @@ class ContainerHandlerSerializer(serializers.Serializer):
     unit_block_id = serializers.CharField(source="unit.location.block_id")
     subsection_location = serializers.CharField(source="subsection.location")
     course_sequence_ids = serializers.ListField(child=serializers.CharField())
+    library_content_picker_url = serializers.CharField()
 
     def get_assets_url(self, obj):
         """

cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py

@@ -131,11 +131,13 @@ def put(self, request: _AuthenticatedRequest, usage_key_string: str) -> Response
         # Note that, if this fails and we raise a 4XX, then we will not call modulstore().update_item,
         # thus preserving the former value of `downstream.upstream`.
         downstream.upstream = new_upstream_ref
-        sync_param = request.data.get("sync", "false").lower()
-        if sync_param not in ["true", "false"]:
+        sync_param = request.data.get("sync", "false")
+        if isinstance(sync_param, str):
+            sync_param = sync_param.lower()
+        if sync_param not in ["true", "false", True, False]:
             raise ValidationError({"sync": "must be 'true' or 'false'"})
         try:
-            if sync_param == "true":
+            if sync_param == "true" or sync_param is True:
                 sync_from_upstream(downstream=downstream, user=request.user)
             else:
                 # Even if we're not syncing (i.e., updating the downstream's values with the upstream's), we still need

cms/djangoapps/contentstore/tests/test_filters.py

@@ -0,0 +1,98 @@
+"""
+Unit tests for the asset upload endpoint.
+"""
+from datetime import datetime
+from urllib.parse import urljoin
+
+from pytz import UTC
+
+from django.test import override_settings
+from cms.djangoapps.contentstore import asset_storage_handlers
+from opaque_keys.edx.locator import CourseLocator
+from openedx_filters import PipelineStep
+from xmodule.contentstore.content import StaticContent
+from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
+
+
+class TestPageURLRequestedPipelineStep(PipelineStep):
+    """
+    Utility class used when getting steps for pipeline.
+    """
+
+    def run_filter(self, url, org):  # pylint: disable=arguments-differ
+        """Pipeline step that modifies lms url creation."""
+        url = "https://lms-url-creation"
+        org = "org"
+        return {
+            "url": url,
+            "org": org,
+        }
+
+
+class LMSPageURLRequestedFiltersTest(ModuleStoreTestCase):
+    """
+    Tests for the Open edX Filters associated with the lms url requested process.
+    This class guarantees that the following filters are triggered during the microsite render:
+    - LMSPageURLRequested
+    """
+
+    def setUp(self):  # pylint: disable=arguments-differ
+        super().setUp()
+        self.upload_date = datetime(2013, 6, 1, 10, 30, tzinfo=UTC)
+        self.content_type = 'image/jpg'
+        self.course_key = CourseLocator('org', 'class', 'run')
+        self.location = self.course_key.make_asset_key('asset', 'my_file_name.jpg')
+        self.thumbnail_location = self.course_key.make_asset_key('thumbnail', 'my_file_name_thumb.jpg')
+
+        self.asset_url = StaticContent.serialize_asset_key_with_slash(self.location)
+
+    @override_settings(
+        OPEN_EDX_FILTERS_CONFIG={
+            "org.openedx.course_authoring.lms.page.url.requested.v1": {
+                "pipeline": [
+                    "common.djangoapps.util.tests.test_filters.TestPageURLRequestedPipelineStep",
+                ],
+                "fail_silently": False,
+            },
+        },
+    )
+    def test_lms_url_requested_filter_executed(self):
+        """
+        Test that filter get new LMS URL for asset URL generation
+        based on the course organization settings for org.
+        Expected result:
+            - LMSPageURLRequested is triggered and executes TestPageURLRequestedPipelineStep.
+            - The arguments that the receiver gets are the arguments used by the filter.
+        """
+        output = asset_storage_handlers.get_asset_json(
+            "my_file",
+            self.content_type,
+            self.upload_date,
+            self.location,
+            self.thumbnail_location,
+            True,
+            self.course_key
+        )
+
+        self.assertEqual(output.get('external_url'), urljoin('https://lms-url-creation', self.asset_url))
+
+    @override_settings(OPEN_EDX_FILTERS_CONFIG={}, LMS_ROOT_URL="https://lms-base")
+    def test_lms_url_requested_without_filter_configuration(self):
+        """
+        Test that filter get new LMS URL for asset URL generation
+        based on LMS_ROOT_URL settings because OPEN_EDX_FILTERS_CONFIG is not set.
+        Expected result:
+            - Returns the asset URL with domain base LMS_ROOT_URL.
+            - The get process ends successfully.
+        """
+        output = asset_storage_handlers.get_asset_json(
+            "my_file",
+            self.content_type,
+            self.upload_date,
+            self.location,
+            self.thumbnail_location,
+            True,
+            self.course_key
+        )
+
+        self.assertEqual(output.get('external_url'), urljoin('https://lms-base', self.asset_url))

cms/djangoapps/contentstore/tests/test_utils.py

@@ -960,4 +960,17 @@ def test_course_update_notification_sent(self):
         send_course_update_notification(self.course.id, content, self.user)
         assert Notification.objects.all().count() == 1
         notification = Notification.objects.first()
-        assert notification.content == "<p><strong><p>content</p></strong></p>"
+        assert notification.content == "<p><strong>content</strong></p>"
+
+    def test_if_content_is_plain_text(self):
+        """
+        Test that the course_update notification is sent.
+        """
+        user = UserFactory()
+        CourseEnrollment.enroll(user=user, course_key=self.course.id)
+        assert Notification.objects.all().count() == 0
+        content = "<p>content<p>Sub content</p><h1>heading</h1></p><img src='' />"
+        send_course_update_notification(self.course.id, content, self.user)
+        assert Notification.objects.all().count() == 1
+        notification = Notification.objects.first()
+        assert notification.content == "<p><strong>content Sub content heading</strong></p>"

cms/djangoapps/contentstore/utils.py

@@ -38,7 +38,28 @@
     exam_setting_view_enabled,
     libraries_v1_enabled,
     libraries_v2_enabled,
+    split_library_view_on_dashboard,
+    use_new_advanced_settings_page,
+    use_new_course_outline_page,
+    use_new_certificates_page,
+    use_new_export_page,
+    use_new_files_uploads_page,
+    use_new_grading_page,
+    use_new_group_configurations_page,
+    use_new_course_team_page,
+    use_new_home_page,
+    use_new_import_page,
+    use_new_schedule_details_page,
+    use_new_text_editor,
+    use_new_textbooks_page,
+    use_new_unit_page,
+    use_new_updates_page,
+    use_new_video_editor,
+    use_new_video_uploads_page,
+    use_new_custom_pages,
 )
+from cms.djangoapps.models.settings.course_grading import CourseGradingModel
+from cms.djangoapps.models.settings.course_metadata import CourseMetadata
 from common.djangoapps.course_action_state.models import CourseRerunUIStateManager, CourseRerunState
 from common.djangoapps.course_action_state.managers import CourseActionStateItemNotFoundError
 from common.djangoapps.course_modes.models import CourseMode
@@ -79,29 +100,6 @@
 from openedx.features.content_type_gating.models import ContentTypeGatingConfig
 from openedx.features.content_type_gating.partitions import CONTENT_TYPE_GATING_SCHEME
 from openedx.features.course_experience.waffle import ENABLE_COURSE_ABOUT_SIDEBAR_HTML
-from cms.djangoapps.contentstore.toggles import (
-    split_library_view_on_dashboard,
-    use_new_advanced_settings_page,
-    use_new_course_outline_page,
-    use_new_certificates_page,
-    use_new_export_page,
-    use_new_files_uploads_page,
-    use_new_grading_page,
-    use_new_group_configurations_page,
-    use_new_course_team_page,
-    use_new_home_page,
-    use_new_import_page,
-    use_new_schedule_details_page,
-    use_new_text_editor,
-    use_new_textbooks_page,
-    use_new_unit_page,
-    use_new_updates_page,
-    use_new_video_editor,
-    use_new_video_uploads_page,
-    use_new_custom_pages,
-)
-from cms.djangoapps.models.settings.course_grading import CourseGradingModel
-from cms.djangoapps.models.settings.course_metadata import CourseMetadata
 from xmodule.library_tools import LegacyLibraryToolsService
 from xmodule.course_block import DEFAULT_START_DATE  # lint-amnesty, pylint: disable=wrong-import-order
 from xmodule.data import CertificatesDisplayBehaviors
@@ -431,6 +429,18 @@ def get_course_outline_url(course_locator) -> str:
     return course_outline_url
 
 
+def get_library_content_picker_url(course_locator) -> str:
+    """
+    Gets course authoring microfrontend library content picker URL for the given parent block.
+    """
+    content_picker_url = None
+    if libraries_v2_enabled():
+        mfe_base_url = get_course_authoring_url(course_locator)
+        content_picker_url = f'{mfe_base_url}/component-picker'
+
+    return content_picker_url
+
+
 def get_unit_url(course_locator, unit_locator) -> str:
     """
     Gets course authoring microfrontend URL for unit page view.
@@ -2045,6 +2055,7 @@ def get_container_handler_context(request, usage_key, course, xblock):  # pylint
         'user_clipboard': user_clipboard,
         'is_fullwidth_content': is_library_xblock,
         'course_sequence_ids': course_sequence_ids,
+        'library_content_picker_url': get_library_content_picker_url(course.id),
     }
     return context
 
@@ -2248,22 +2259,9 @@ def clean_html_body(html_body):
     Get html body, remove tags and limit to 500 characters
     """
     html_body = BeautifulSoup(Truncator(html_body).chars(500, html=True), 'html.parser')
-
-    tags_to_remove = [
-        "a", "link",  # Link Tags
-        "img", "picture", "source",  # Image Tags
-        "video", "track",  # Video Tags
-        "audio",  # Audio Tags
-        "embed", "object", "iframe",  # Embedded Content
-        "script"
-    ]
-
-    # Remove the specified tags while keeping their content
-    for tag in tags_to_remove:
-        for match in html_body.find_all(tag):
-            match.unwrap()
-
-    return str(html_body)
+    text_content = html_body.get_text(separator=" ").strip()
+    text_content = text_content.replace('\n', '').replace('\r', '')
+    return text_content
 
 
 def send_course_update_notification(course_key, content, user):

cms/djangoapps/contentstore/views/block.py

@@ -135,6 +135,8 @@ def xblock_handler(request, usage_key_string=None):
                      if duplicate_source_locator is not present
                 :staged_content: use "clipboard" to paste from the OLX user's clipboard. (Incompatible with all other
                      fields except parent_locator)
+                :library_content_key: the key of the library content to add. (Incompatible with
+                     all other fields except parent_locator)
               The locator (unicode representation of a UsageKey) for the created xblock (minus children) is returned.
     """
     return handle_xblock(request, usage_key_string)