PT-14471: Extended XML model with Images

feat: Extends category and product provider to load images and attach them to sitemap using the correct google sitemap structure. Image sitemaps are a good way of telling Google about other images on your site, especially those that we might not otherwise find (such as images your site reaches with JavaScript code). You can create a separate image sitemap or add image sitemap tags to your existing sitemap; either approach is equally fine for Google. https://developers.google.com/search/docs/crawling-indexing/sitemaps/image-sitemaps
VirtoCommerce · Nov 14, 2023 · 5447fa6 · 5447fa6
1 parent 7d72b1b
commit 5447fa6
Show file tree

Hide file tree

Showing 12 changed files with 306 additions and 98 deletions.
diff --git a/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItem.cs b/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItem.cs
@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
 using System.Linq;
 using VirtoCommerce.Platform.Core.Common;
 
@@ -27,6 +26,7 @@ public class SitemapItem : AuditableEntity, ICloneable
         public virtual object Clone()
         {
             var result = MemberwiseClone() as SitemapItem;
+
             result.ItemsRecords = ItemsRecords?.Select(x => x.Clone()).OfType<SitemapItemRecord>().ToList();
 
             return result;

diff --git a/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItemImageRecord.cs b/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItemImageRecord.cs
@@ -0,0 +1,18 @@
+using System;
+
+namespace VirtoCommerce.SitemapsModule.Core.Models
+{
+    public class SitemapItemImageRecord : ICloneable
+    {
+        public string Loc { get; set; }
+
+        #region ICloneable members
+
+        public virtual object Clone()
+        {
+            return MemberwiseClone() as SitemapItemImageRecord;
+        }
+
+        #endregion
+    }
+}
diff --git a/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItemRecord.cs b/src/VirtoCommerce.SitemapsModule.Core/Models/SitemapItemRecord.cs
@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.ObjectModel;
 using System.Linq;
 
 namespace VirtoCommerce.SitemapsModule.Core.Models
@@ -17,13 +16,16 @@ public class SitemapItemRecord : ICloneable
 
         public ICollection<SitemapItemAlternateLinkRecord> Alternates { get; set; } = new List<SitemapItemAlternateLinkRecord>();
 
+        public ICollection<SitemapItemImageRecord> Images { get; set; } = new List<SitemapItemImageRecord>();
+
         #region ICloneable members
 
         public virtual object Clone()
         {
             var result = MemberwiseClone() as SitemapItemRecord;
 
             result.Alternates = Alternates?.Select(x => x.Clone()).OfType<SitemapItemAlternateLinkRecord>().ToList();
+            result.Images = Images?.Select(x => x.Clone()).OfType<SitemapItemImageRecord>().ToList();
 
             return result;
         }

diff --git a/src/VirtoCommerce.SitemapsModule.Core/ModuleConstants.cs b/src/VirtoCommerce.SitemapsModule.Core/ModuleConstants.cs
@@ -60,6 +60,14 @@ public static class General
                     DefaultValue = ".md,.html"
                 };
 
+                public static readonly SettingDescriptor IncludeImages = new SettingDescriptor
+                {
+                    Name = "Sitemap.IncludeImages",
+                    GroupName = "Sitemap|General",
+                    ValueType = SettingValueType.Boolean,
+                    DefaultValue = false
+                };
+
                 public static IEnumerable<SettingDescriptor> AllSettings
                 {
                     get
@@ -68,6 +76,7 @@ public static IEnumerable<SettingDescriptor> AllSettings
                         yield return FilenameSeparator;
                         yield return SearchBunchSize;
                         yield return AcceptedFilenameExtensions;
+                        yield return IncludeImages;
                     }
                 }
             }

diff --git a/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapItemImageXmlRecord.cs b/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapItemImageXmlRecord.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Xml.Serialization;
+using VirtoCommerce.SitemapsModule.Core.Models;
+
+namespace VirtoCommerce.SitemapsModule.Data.Models.Xml
+{
+    [Serializable]
+    [XmlType(Namespace = "http://www.google.com/schemas/sitemap-image/1.1")]
+    public class SitemapItemImageXmlRecord
+    {
+        [XmlElement("loc")]
+        public string Loc { get; set; }
+
+        public virtual SitemapItemImageXmlRecord ToXmlModel(SitemapItemImageRecord coreModel)
+        {
+            if (coreModel == null)
+            {
+                throw new ArgumentNullException(nameof(coreModel));
+            }
+
+            Loc = coreModel.Loc;
+
+            return this;
+        }
+    }
+}
diff --git a/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapItemXmlRecord.cs b/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapItemXmlRecord.cs
@@ -24,6 +24,12 @@ public class SitemapItemXmlRecord
         [XmlElement("link", Namespace = "http://www.w3.org/1999/xhtml")]
         public List<SitemapItemAlternateLinkXmlRecord> Alternates { get; set; }
 
+        /// <summary>
+        /// Containes images if load images checkbox is activated in settings
+        /// </summary>
+        [XmlElement("image", Namespace = "http://www.google.com/schemas/sitemap-image/1.1")]
+        public List<SitemapItemImageXmlRecord> Images { get; set; }
+
         public virtual SitemapItemXmlRecord ToXmlModel(SitemapItemRecord coreModel)
         {
             if (coreModel == null)
@@ -36,6 +42,7 @@ public virtual SitemapItemXmlRecord ToXmlModel(SitemapItemRecord coreModel)
             UpdateFrequency = coreModel.UpdateFrequency;
             Url = coreModel.Url;
             Alternates = coreModel.Alternates.Count > 0 ? coreModel.Alternates.Select(a => (new SitemapItemAlternateLinkXmlRecord()).ToXmlModel(a)).ToList() : null;
+            Images = coreModel.Images.Count > 0 ? coreModel.Images.Select(a => (new SitemapItemImageXmlRecord()).ToXmlModel(a)).ToList() : null;
 
             return this;
         }

diff --git a/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapXmlRecord.cs b/src/VirtoCommerce.SitemapsModule.Data/Models/Xml/SitemapXmlRecord.cs
@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Xml.Serialization;
 
@@ -13,7 +13,13 @@ public SitemapXmlRecord()
             Items = new List<SitemapItemXmlRecord>();
         }
 
+        /// <summary>
+        /// Property that is used to dynamically set xml namespaces for objects like Image
+        /// </summary>
+        [XmlNamespaceDeclarations]
+        public XmlSerializerNamespaces xmlns;
+
         [XmlElement("url")]
         public List<SitemapItemXmlRecord> Items { get; set; }
     }
-}
+}
diff --git a/...temapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs b/...temapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs
@@ -1,7 +1,9 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using VirtoCommerce.CatalogModule.Core.Model;
+using VirtoCommerce.CatalogModule.Core.Model.ListEntry;
 using VirtoCommerce.CatalogModule.Core.Model.Search;
 using VirtoCommerce.CatalogModule.Core.Search;
 using VirtoCommerce.CatalogModule.Core.Services;
@@ -20,17 +22,24 @@ public class CatalogSitemapItemRecordProvider : SitemapItemRecordProviderBase, I
         private readonly ISettingsManager _settingsManager;
         private readonly IItemService _itemService;
         private readonly IListEntrySearchService _listEntrySearchService;
+        private readonly IProductSearchService _productSearchService;
+        private readonly ICategorySearchService _categorySearchService;
 
         public CatalogSitemapItemRecordProvider(
             ISitemapUrlBuilder urlBuilder,
             ISettingsManager settingsManager,
             IItemService itemService,
-            IListEntrySearchService listEntrySearchService)
+            IListEntrySearchService listEntrySearchService,
+            IProductSearchService productSearchService,
+            ICategorySearchService categorySearchService)
             : base(urlBuilder)
         {
             _settingsManager = settingsManager;
             _itemService = itemService;
             _listEntrySearchService = listEntrySearchService;
+
+            _productSearchService = productSearchService;
+            _categorySearchService = categorySearchService;
         }
 
         #region ISitemapItemRecordProvider members
@@ -44,13 +53,17 @@ public virtual async Task LoadSitemapItemRecordsAsync(Store store, Sitemap sitem
             {
                 throw new ArgumentNullException(nameof(sitemap));
             }
+
             await LoadCategoriesSitemapItemRecordsAsync(store, sitemap, baseUrl, progressCallback);
             await LoadProductsSitemapItemRecordsAsync(store, sitemap, baseUrl, progressCallback);
         }
+
         #endregion
 
         protected virtual async Task LoadCategoriesSitemapItemRecordsAsync(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
         {
+            var shouldIncludeImages = await _settingsManager.GetValueAsync<bool>(ModuleConstants.Settings.General.IncludeImages);
+
             var progressInfo = new ExportImportProgressInfo();
             var categoryOptions = await GetCategoryOptions(store);
             var batchSize = await _settingsManager.GetValueAsync<int>(ModuleConstants.Settings.General.SearchBunchSize);
@@ -77,9 +90,35 @@ protected virtual async Task LoadCategoriesSitemapItemRecordsAsync(Store store,
                         var result = await _listEntrySearchService.SearchAsync(listEntrySearchCriteria);
                         totalCount = result.TotalCount;
                         listEntrySearchCriteria.Skip += batchSize;
+
+                        // Only used if should include images
+                        List<CatalogProduct> products = new List<CatalogProduct>();
+                        if (shouldIncludeImages)
+                        {
+                            // If images need to be included - run a search for picked products to get variations with images
+                            var productIds = result.Results.Where(x => x is ProductListEntry).Select(x => x.Id).ToArray();
+                            products = await SearchProductsWithVariations(batchSize, productIds);
+                        }
+
                         foreach (var listEntry in result.Results)
                         {
-                            categorySiteMapItem.ItemsRecords.AddRange(GetSitemapItemRecords(store, categoryOptions, sitemap.UrlTemplate, baseUrl, listEntry));
+                            var itemRecords = GetSitemapItemRecords(store, categoryOptions, sitemap.UrlTemplate, baseUrl, listEntry).ToList();
+
+                            if (shouldIncludeImages && listEntry is ProductListEntry)
+                            {
+                                var item = products.FirstOrDefault(x => x.Id == listEntry.Id);
+
+                                // for each record per product add image urls to sitemap
+                                foreach (var record in itemRecords)
+                                {
+                                    record.Images.AddRange(item.Images.Select(x => new SitemapItemImageRecord
+                                    {
+                                        Loc = x.Url
+                                    }));
+                                }
+                            }
+
+                            categorySiteMapItem.ItemsRecords.AddRange(itemRecords);
                         }
                         progressInfo.Description = $"Catalog: Have been generated  {Math.Min(listEntrySearchCriteria.Skip, totalCount)} of {totalCount} records for category {categorySiteMapItem.Title} item";
                         progressCallback?.Invoke(progressInfo);
@@ -90,38 +129,114 @@ protected virtual async Task LoadCategoriesSitemapItemRecordsAsync(Store store,
             }
         }
 
+        /// <summary>
+        /// This helps keeping the imageless flow untouched for products
+        /// </summary>
+        /// <param name="store"></param>
+        /// <param name="sitemap"></param>
+        /// <param name="baseUrl"></param>
+        /// <param name="progressCallback"></param>
+        /// <returns></returns>
         protected virtual async Task LoadProductsSitemapItemRecordsAsync(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
         {
-            var progressInfo = new ExportImportProgressInfo();
-            var productOptions = await GetProductOptions(store);
+            var shouldIncludeImages = await _settingsManager.GetValueAsync<bool>(ModuleConstants.Settings.General.IncludeImages);
+
+            if (shouldIncludeImages)
+            {
+                await LoadProductsWithoutImages(store, sitemap, baseUrl, progressCallback);
+            }
+            else
+            {
+                await LoadProductsWithImages(store, sitemap, baseUrl, progressCallback);
+            }
+        }
+
+        private async Task<List<CatalogProduct>> SearchProductsWithVariations(int batchSize, string[] productIds = null, ProductSearchCriteria searchCriteria = null, Action<ExportImportProgressInfo> progressCallback = null)
+        {
+            var products = (await _itemService.GetAsync(productIds, (ItemResponseGroup.Seo | ItemResponseGroup.Outlines | ItemResponseGroup.WithImages).ToString()))
+    .           Where(p => !p.IsActive.HasValue || p.IsActive.Value).ToList();
+
+            return products;
+        }
+
+        /// <summary>
+        /// This is used to load products with images
+        /// Images are attached to corresponding product per item record
+        /// </summary>
+        /// <param name="store"></param>
+        /// <param name="sitemap"></param>
+        /// <param name="baseUrl"></param>
+        /// <param name="progressCallback"></param>
+        /// <returns></returns>
+        private async Task LoadProductsWithImages(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
+        {
             var batchSize = await _settingsManager.GetValueAsync<int>(ModuleConstants.Settings.General.SearchBunchSize);
 
-            var skip = 0;
             var productSitemapItems = sitemap.Items.Where(x => x.ObjectType.EqualsInvariant(SitemapItemTypes.Product)).ToList();
-            if (productSitemapItems.Count > 0)
-            {
-                progressInfo.Description = $"Catalog: Starting records generation  for {productSitemapItems.Count} products items";
-                progressCallback?.Invoke(progressInfo);
 
-                do
+            var productOptions = await GetProductOptions(store);
+
+            var productIds = productSitemapItems.Select(x => x.ObjectId).ToArray();
+
+            var products = await SearchProductsWithVariations(batchSize, productIds);
+
+            foreach (var product in products)
+            {
+                var productSitemapItem = productSitemapItems.FirstOrDefault(x => x.ObjectId.EqualsInvariant(product.Id));
+                if (productSitemapItem != null)
                 {
-                    var productIds = productSitemapItems.Select(x => x.ObjectId).Skip(skip).Take(batchSize).ToArray();
-                    var products = (await _itemService.GetAsync(productIds, (ItemResponseGroup.Seo | ItemResponseGroup.Outlines).ToString())).Where(p => !p.IsActive.HasValue || p.IsActive.Value);
-                    skip += batchSize;
-                    foreach (var product in products)
+                    var itemRecords = GetSitemapItemRecords(store, productOptions, sitemap.UrlTemplate, baseUrl, product);
+
+                    foreach (var item in itemRecords)
                     {
-                        var productSitemapItem = productSitemapItems.FirstOrDefault(x => x.ObjectId.EqualsInvariant(product.Id));
-                        if (productSitemapItem != null)
+                        var existingImages = product.Images.Where(x => !string.IsNullOrWhiteSpace(x.Url)).ToList();
+                        if (existingImages.Count > 0)
                         {
-                            var itemRecords = GetSitemapItemRecords(store, productOptions, sitemap.UrlTemplate, baseUrl, product);
-                            productSitemapItem.ItemsRecords.AddRange(itemRecords);
+                            item.Images.AddRange(existingImages.Select(x => new SitemapItemImageRecord
+                            {
+                                Loc = x.Url
+                            }));
                         }
                     }
-                    progressInfo.Description = $"Catalog: Have been generated  {Math.Min(skip, productSitemapItems.Count)} of {productSitemapItems.Count} records for products items";
-                    progressCallback?.Invoke(progressInfo);
+
+                    productSitemapItem.ItemsRecords.AddRange(itemRecords);
+                }
+            }
+        }
+
+        private async Task LoadProductsWithoutImages(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
+        {
+            var batchSize = await _settingsManager.GetValueAsync<int>(ModuleConstants.Settings.General.SearchBunchSize);
+
+            var productSitemapItems = sitemap.Items.Where(x => x.ObjectType.EqualsInvariant(SitemapItemTypes.Product)).ToList();
+            var skip = 0;
+            var productOptions = await GetProductOptions(store);
+
+            var progressInfo = new ExportImportProgressInfo();
+
+            do
+            {
+                var productIds = productSitemapItems.Select(x => x.ObjectId).Skip(skip).Take(batchSize).ToArray();
+
+                var products = (await _itemService.GetAsync(productIds, (ItemResponseGroup.Seo | ItemResponseGroup.Outlines ).ToString()))
+                    .Where(p => !p.IsActive.HasValue || p.IsActive.Value);
+
+                skip += batchSize;
+
+                foreach (var product in products)
+                {
+                    var productSitemapItem = productSitemapItems.FirstOrDefault(x => x.ObjectId.EqualsInvariant(product.Id));
+                    if (productSitemapItem != null)
+                    {
+                        var itemRecords = GetSitemapItemRecords(store, productOptions, sitemap.UrlTemplate, baseUrl, product);
+
+                        productSitemapItem.ItemsRecords.AddRange(itemRecords);
+                    }
                 }
-                while (skip < productSitemapItems.Count);
+                progressInfo.Description = $"Catalog: Have been generated  {Math.Min(skip, productSitemapItems.Count)} of {productSitemapItems.Count} records for products items";
+                progressCallback?.Invoke(progressInfo);
             }
+            while (skip < productSitemapItems.Count);
         }
 
         private async Task<SitemapItemOptions> GetProductOptions(Store store)