diff --git a/Directory.Build.props b/Directory.Build.props index 881b658..e7502ac 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,7 +2,7 @@ - 3.401.0 + 3.402.0 $(VersionSuffix)-$(BuildNumber) diff --git a/src/VirtoCommerce.SitemapsModule.Data/Services/ISitemapXmlGenerator.cs b/src/VirtoCommerce.SitemapsModule.Data/Services/ISitemapXmlGenerator.cs index aca7feb..10eac49 100644 --- a/src/VirtoCommerce.SitemapsModule.Data/Services/ISitemapXmlGenerator.cs +++ b/src/VirtoCommerce.SitemapsModule.Data/Services/ISitemapXmlGenerator.cs @@ -8,7 +8,7 @@ namespace VirtoCommerce.SitemapsModule.Data.Services { public interface ISitemapXmlGenerator { - Task> GetSitemapUrlsAsync(string storeId); + Task> GetSitemapUrlsAsync(string storeId, string baseUrl); Task GenerateSitemapXmlAsync(string storeId, string baseUrl, string sitemapUrl, Action progressCallback = null); } diff --git a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs index 076d08c..f17cab0 100644 --- a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs +++ b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs @@ -90,7 +90,7 @@ protected virtual async Task LoadCategoriesSitemapItemRecordsAsync(Store store, { // If images need to be included - run a search for picked products to get variations with images var productIds = result.Results.Where(x => x is ProductListEntry).Select(x => x.Id).ToArray(); - products = await SearchProductsWithVariations(batchSize, productIds); + products = await SearchProductsWithVariations(productIds); } foreach (var listEntry in result.Results) @@ -147,7 +147,7 @@ protected virtual async Task LoadProductsSitemapItemRecordsAsync(Store store, Si } } - private async Task> SearchProductsWithVariations(int batchSize, string[] productIds = null, ProductSearchCriteria searchCriteria = null, Action progressCallback = null) + private async Task> SearchProductsWithVariations(string[] productIds = null) { var products = (await _itemService.GetAsync(productIds, (ItemResponseGroup.Seo | ItemResponseGroup.Outlines | ItemResponseGroup.WithImages).ToString())) .Where(p => !p.IsActive.HasValue || p.IsActive.Value).ToList(); @@ -166,15 +166,17 @@ private async Task> SearchProductsWithVariations(int batchS /// private async Task LoadProductsWithImages(Store store, Sitemap sitemap, string baseUrl, Action progressCallback = null) { - var batchSize = await _settingsManager.GetValueAsync(ModuleConstants.Settings.General.SearchBunchSize); - var productSitemapItems = sitemap.Items.Where(x => x.ObjectType.EqualsInvariant(SitemapItemTypes.Product)).ToList(); var productOptions = GetProductOptions(store); var productIds = productSitemapItems.Select(x => x.ObjectId).ToArray(); - var products = await SearchProductsWithVariations(batchSize, productIds); + var products = await SearchProductsWithVariations(productIds); + + var progressInfo = new ExportImportProgressInfo(); + + var count = 0; foreach (var product in products) { @@ -197,6 +199,10 @@ private async Task LoadProductsWithImages(Store store, Sitemap sitemap, string b productSitemapItem.ItemsRecords.AddRange(itemRecords); } + + count++; + progressInfo.Description = $"Catalog: Have been generated {count} of {products.Count} records for products items"; + progressCallback?.Invoke(progressInfo); } } diff --git a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/StaticContentSitemapItemRecordProvider.cs b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/StaticContentSitemapItemRecordProvider.cs index 92707a0..b3a2a72 100644 --- a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/StaticContentSitemapItemRecordProvider.cs +++ b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapItemRecordProviders/StaticContentSitemapItemRecordProvider.cs @@ -5,6 +5,7 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using VirtoCommerce.AssetsModule.Core.Assets; +using VirtoCommerce.ContentModule.Core.Model; using VirtoCommerce.ContentModule.Core.Services; using VirtoCommerce.Platform.Core.Common; using VirtoCommerce.Platform.Core.ExportImport; @@ -13,95 +14,81 @@ using VirtoCommerce.SitemapsModule.Core.Models; using VirtoCommerce.SitemapsModule.Core.Services; using VirtoCommerce.SitemapsModule.Data.Extensions; -using VirtoCommerce.StoreModule.Core.Model; using VirtoCommerce.Tools; using YamlDotNet.RepresentationModel; +using Store = VirtoCommerce.StoreModule.Core.Model.Store; namespace VirtoCommerce.SitemapsModule.Data.Services.SitemapItemRecordProviders { public class StaticContentSitemapItemRecordProvider : SitemapItemRecordProviderBase, ISitemapItemRecordProvider { - private readonly ISettingsManager _settingsManager; - private readonly IBlobContentStorageProviderFactory _blobStorageProviderFactory; + private const string PagesContentType = "pages"; private static readonly Regex _headerRegExp = new Regex(@"(?s:^---(.*?)---)"); + private readonly ISettingsManager _settingsManager; + private readonly IContentFileService _contentFileService; + private readonly IContentService _contentService; + public StaticContentSitemapItemRecordProvider( ISitemapUrlBuilder urlBuilder, ISettingsManager settingsManager, - IBlobContentStorageProviderFactory blobStorageProviderFactory) + IContentService contentService, + IContentFileService contentFileService) : base(urlBuilder) { _settingsManager = settingsManager; - _blobStorageProviderFactory = blobStorageProviderFactory; + _contentService = contentService; + _contentFileService = contentFileService; } public virtual async Task LoadSitemapItemRecordsAsync(Store store, Sitemap sitemap, string baseUrl, Action progressCallback = null) { var progressInfo = new ExportImportProgressInfo(); - var contentBasePath = $"Pages/{sitemap.StoreId}"; - var storageProvider = _blobStorageProviderFactory.CreateProvider(contentBasePath); - - var staticContentSitemapItems = sitemap.Items - .Where(si => !string.IsNullOrEmpty(si.ObjectType)) - .Where(si => si.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) || si.ObjectType.EqualsInvariant(SitemapItemTypes.Folder)) - .ToList(); + var staticContentSitemapItems = GetStaticContentSitemapItems(sitemap); var totalCount = staticContentSitemapItems.Count; - if (totalCount <= 0) + if (totalCount == 0) { return; } var processedCount = 0; - var acceptedFilenameExtensions = (await _settingsManager.GetValueAsync(ModuleConstants.Settings.General.AcceptedFilenameExtensions)) - .Split(',') - .Select(i => i.Trim()) - .Where(i => !string.IsNullOrEmpty(i)) - .ToList(); + var allowedExtensions = await GetAllowedExtensions(); + var blogOptions = await GetBlogOptions(store); progressInfo.Description = $"Content: Starting records generation for {totalCount} pages"; progressCallback?.Invoke(progressInfo); foreach (var sitemapItem in staticContentSitemapItems) { - var urls = new List(); + var validSitemapItems = new List(); + if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.Folder)) { - var searchResult = await storageProvider.SearchAsync(sitemapItem.UrlTemplate, null); - - if (searchResult.TotalCount == 0) - { - searchResult = await storageProvider.SearchAsync("blogs/" + sitemapItem.UrlTemplate, null); - } - - var itemUrls = await GetItemUrls(storageProvider, searchResult); - foreach (var itemUrl in itemUrls.Where(itemUrl => IsExtensionAllowed(acceptedFilenameExtensions, itemUrl))) - { - urls.Add(itemUrl); - } + await LoadPagesRecursivly(sitemap.StoreId, sitemapItem.UrlTemplate, allowedExtensions, validSitemapItems); } - else if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem)) + else if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) && + IsExtensionAllowed(allowedExtensions, sitemapItem.UrlTemplate) && + await _contentService.ItemExistsAsync(PagesContentType, sitemap.StoreId, sitemapItem.UrlTemplate)) { - var item = await storageProvider.GetBlobInfoAsync(sitemapItem.UrlTemplate); - if (item != null && IsExtensionAllowed(acceptedFilenameExtensions, item.RelativeUrl)) - { - urls.Add(item.RelativeUrl); - } + validSitemapItems.Add(sitemapItem.UrlTemplate); } - totalCount = urls.Count; + totalCount = validSitemapItems.Count; - foreach (var url in urls) + foreach (var url in validSitemapItems) { - using (var stream = await storageProvider.OpenReadAsync(url)) + var contentFile = await _contentService.GetFileAsync(PagesContentType, sitemap.StoreId, url); + + using (var stream = await _contentService.GetItemStreamAsync(PagesContentType, sitemap.StoreId, url)) { var content = stream.ReadToString(); - var frontMatterPermalink = GetPermalink(content, url); - frontMatterPermalink.FilePath = url; + + var frontMatterPermalink = GetPermalink(content, url, contentFile.Url); var urlTemplate = frontMatterPermalink.ToUrl().TrimStart('/'); - var blogOptions = await GetBlogOptions(store); + var records = GetSitemapItemRecords(store, blogOptions, urlTemplate, baseUrl); sitemapItem.ItemsRecords.AddRange(records); } @@ -113,6 +100,43 @@ public virtual async Task LoadSitemapItemRecordsAsync(Store store, Sitemap sitem } } + private async Task LoadPagesRecursivly(string storeId, string folrderUrl, List allowedExtensions, List validSitemapItems) + { + var criteria = AbstractTypeFactory.TryCreateInstance(); + criteria.ContentType = PagesContentType; + criteria.StoreId = storeId; + criteria.FolderUrl = folrderUrl; + + var searchResult = await _contentFileService.FilterItemsAsync(criteria); + + foreach (var file in searchResult.Where(file => file.Type == "blob" && IsExtensionAllowed(allowedExtensions, file.RelativeUrl))) + { + validSitemapItems.Add(file.RelativeUrl); + } + + // Load Pages from SubFolders + foreach (var folder in searchResult.Where(file => file.Type == "folder")) + { + await LoadPagesRecursivly(storeId, folder.RelativeUrl, allowedExtensions, validSitemapItems); + } + } + + private static List GetStaticContentSitemapItems(Sitemap sitemap) + { + return sitemap.Items + .Where(si => !string.IsNullOrEmpty(si.ObjectType)) + .Where(si => si.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) || si.ObjectType.EqualsInvariant(SitemapItemTypes.Folder)) + .ToList(); + } + + private async Task> GetAllowedExtensions() + { + return (await _settingsManager.GetValueAsync(ModuleConstants.Settings.General.AcceptedFilenameExtensions)) + .Split(',') + .Select(i => i.Trim()) + .Where(i => !string.IsNullOrEmpty(i)) + .ToList(); + } private async Task GetBlogOptions(Store store) { @@ -142,7 +166,7 @@ private static bool IsExtensionAllowed(IList acceptedFilenameExtensions, return string.IsNullOrEmpty(itemExtension) || acceptedFilenameExtensions.Contains(itemExtension, StringComparer.OrdinalIgnoreCase); } - private static FrontMatterPermalink GetPermalink(string content, string url) + private static FrontMatterPermalink GetPermalink(string content, string url, string filePath) { if (content.TryParseJson(out var token) && token.HasValues && token.First?["permalink"] != null) { @@ -153,10 +177,10 @@ private static FrontMatterPermalink GetPermalink(string content, string url) yamlHeader.TryGetValue("permalink", out var permalinks); if (permalinks != null) { - return new FrontMatterPermalink(permalinks.FirstOrDefault()); + return new FrontMatterPermalink(permalinks.FirstOrDefault()) { FilePath = filePath }; } - return new FrontMatterPermalink(url.Replace(".md", "")); + return new FrontMatterPermalink(url.Replace(".md", "")) { FilePath = filePath }; } private static async Task> GetItemUrls(IBlobContentStorageProvider storageProvider, GenericSearchResult searchResult) @@ -224,3 +248,4 @@ private static IEnumerable GetYamlNodeValues(YamlNode value) } } } +; diff --git a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapUrlBuilder.cs b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapUrlBuilder.cs index 3eef0bb..1fe2439 100644 --- a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapUrlBuilder.cs +++ b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapUrlBuilder.cs @@ -26,6 +26,10 @@ public SitemapUrlBuilder(IUrlBuilder urlBuilder) public virtual string BuildStoreUrl(Store store, string language, string urlTemplate, string baseUrl, IEntity entity = null) { var toolsStore = store.ToToolsStore(baseUrl); + if (!string.IsNullOrEmpty(baseUrl)) + { + toolsStore.Url = baseUrl; + } var seoSupport = entity as ISeoSupport; diff --git a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapXmlGenerator.cs b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapXmlGenerator.cs index c71dd40..c610bb1 100644 --- a/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapXmlGenerator.cs +++ b/src/VirtoCommerce.SitemapsModule.Data/Services/SitemapXmlGenerator.cs @@ -46,7 +46,7 @@ public SitemapXmlGenerator( _storeService = storeService; } - public virtual async Task> GetSitemapUrlsAsync(string storeId) + public virtual async Task> GetSitemapUrlsAsync(string storeId, string baseUrl) { if (string.IsNullOrEmpty(storeId)) { @@ -56,7 +56,7 @@ public virtual async Task> GetSitemapUrlsAsync(string storeI var sitemapUrls = new List(); var store = await _storeService.GetByIdAsync(storeId, StoreResponseGroup.StoreInfo.ToString()); - var sitemaps = await LoadAllStoreSitemaps(store, ""); + var sitemaps = await LoadStoreSitemaps(store, baseUrl); foreach (var sitemap in sitemaps) { sitemapUrls.AddRange(sitemap.PagedLocations); @@ -73,7 +73,6 @@ public virtual async Task GenerateSitemapXmlAsync(string storeId, string var recordsLimitPerFile = await _settingsManager.GetValueAsync(ModuleConstants.Settings.General.RecordsLimitPerFile); var xmlNamespaces = new XmlSerializerNamespaces(); - //xmlNamespaces.Add("", "https://www.sitemaps.org/schemas/sitemap/0.9"); xmlNamespaces.Add("xhtml", "https://www.w3.org/1999/xhtml"); xmlNamespaces.Add("image", "http://www.google.com/schemas/sitemap-image/1.1"); @@ -86,15 +85,14 @@ public virtual async Task GenerateSitemapXmlAsync(string storeId, string Description = "Creating sitemap.xml..." }); - var allStoreSitemaps = await LoadAllStoreSitemaps(store, baseUrl); + var storeSitemaps = await LoadStoreSitemaps(store, baseUrl); var sitemapIndexXmlRecord = new SitemapIndexXmlRecord(); - foreach (var sitemap in allStoreSitemaps) + foreach (var sitemap in storeSitemaps) { var xmlSiteMapRecords = sitemap.PagedLocations.Select(location => new SitemapIndexItemXmlRecord { - //ModifiedDate = sitemap.Items.Select(x => x.ModifiedDate).OrderByDescending(x => x).FirstOrDefault()?.ToString("yyyy-MM-dd"), Url = _sitemapUrlBuilder.BuildStoreUrl(store, store.DefaultLanguage, location, baseUrl), }).ToList(); @@ -112,7 +110,7 @@ public virtual async Task GenerateSitemapXmlAsync(string storeId, string { await LoadSitemapRecords(store, sitemap, baseUrl, progressCallback); - var distinctRecords = sitemap.Items.SelectMany(x => x.ItemsRecords);//.GroupBy(x => x.Url).Select(x => x.FirstOrDefault()); + var distinctRecords = sitemap.Items.SelectMany(x => x.ItemsRecords); var sitemapItemRecords = distinctRecords.Skip((sitemapLocation.PageNumber - 1) * recordsLimitPerFile).Take(recordsLimitPerFile).ToArray(); var sitemapRecord = new SitemapXmlRecord @@ -132,17 +130,18 @@ public virtual async Task GenerateSitemapXmlAsync(string storeId, string return stream; } - private async Task> LoadAllStoreSitemaps(Store store, string baseUrl) + private async Task> LoadStoreSitemaps(Store store, string baseUrl) { - var sitemaps = new List(); var sitemapSearchCriteria = new SitemapSearchCriteria { StoreId = store.Id, Skip = 0, Take = int.MaxValue }; + var sitemapSearchResult = await _sitemapSearchService.SearchAsync(sitemapSearchCriteria); + var sitemaps = new List(); foreach (var sitemap in sitemapSearchResult.Results) { await LoadSitemapRecords(store, sitemap, baseUrl); @@ -166,7 +165,6 @@ private async Task LoadSitemapRecords(Store store, Sitemap sitemap, string baseU }; sitemap.Items = (await _sitemapItemSearchService.SearchAsync(sitemapItemSearchCriteria)).Results; - var imageUrls = new List(); foreach (var recordProvider in _sitemapItemRecordProviders) { //Log exceptions to prevent fail whole sitemap.xml generation diff --git a/src/VirtoCommerce.SitemapsModule.Web/Controllers/Api/SitemapsModuleApiController.cs b/src/VirtoCommerce.SitemapsModule.Web/Controllers/Api/SitemapsModuleApiController.cs index 470b661..efca849 100644 --- a/src/VirtoCommerce.SitemapsModule.Web/Controllers/Api/SitemapsModuleApiController.cs +++ b/src/VirtoCommerce.SitemapsModule.Web/Controllers/Api/SitemapsModuleApiController.cs @@ -11,8 +11,8 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; -using VirtoCommerce.Platform.Core; using VirtoCommerce.AssetsModule.Core.Assets; +using VirtoCommerce.Platform.Core; using VirtoCommerce.Platform.Core.Common; using VirtoCommerce.Platform.Core.Exceptions; using VirtoCommerce.Platform.Core.ExportImport; @@ -72,7 +72,7 @@ public SitemapsModuleApiController( IPushNotificationManager notifier, IBlobStorageProvider blobStorageProvider, IBlobUrlResolver blobUrlResolver, - IWebHostEnvironment hostingEnvironment, + IWebHostEnvironment hostingEnvironment, IOptions platformOptions) { _sitemapService = sitemapService; @@ -276,7 +276,7 @@ public async Task> GetSitemapsSchema(string storeId) return BadRequest("storeId is empty"); } - var sitemapUrls = await _sitemapXmlGenerator.GetSitemapUrlsAsync(storeId); + var sitemapUrls = await _sitemapXmlGenerator.GetSitemapUrlsAsync(storeId, string.Empty); return Ok(sitemapUrls); } @@ -379,7 +379,7 @@ void SendNotificationWithProgressInfo(ExportImportProgressInfo c) // Create default sitemap.xml await CreateSitemapPartAsync(zipArchive, storeId, baseUrl, "sitemap.xml", SendNotificationWithProgressInfo); - var sitemapUrls = await _sitemapXmlGenerator.GetSitemapUrlsAsync(storeId); + var sitemapUrls = await _sitemapXmlGenerator.GetSitemapUrlsAsync(storeId, baseUrl); foreach (var sitemapUrl in sitemapUrls.Where(url => !string.IsNullOrEmpty(url))) { await CreateSitemapPartAsync(zipArchive, storeId, baseUrl, sitemapUrl, SendNotificationWithProgressInfo); @@ -390,7 +390,7 @@ void SendNotificationWithProgressInfo(ExportImportProgressInfo c) using (var localStream = SystemFile.Open(localTmpPath, FileMode.Open, FileAccess.Read)) using (var blobStream = _blobStorageProvider.OpenWrite(relativeUrl)) { - localStream.CopyTo(blobStream); + await localStream.CopyToAsync(blobStream); } // Add unique key for every link to prevent browser caching @@ -415,7 +415,7 @@ private async Task CreateSitemapPartAsync(ZipArchive zipArchive, string storeId, using (var sitemapPartStream = sitemapPart.Open()) { var stream = await _sitemapXmlGenerator.GenerateSitemapXmlAsync(storeId, baseUrl, sitemapUrl, progressCallback); - stream.CopyTo(sitemapPartStream); + await stream.CopyToAsync(sitemapPartStream); } } } diff --git a/src/VirtoCommerce.SitemapsModule.Web/Module.cs b/src/VirtoCommerce.SitemapsModule.Web/Module.cs index 382e7f9..2bca984 100644 --- a/src/VirtoCommerce.SitemapsModule.Web/Module.cs +++ b/src/VirtoCommerce.SitemapsModule.Web/Module.cs @@ -98,17 +98,17 @@ public void Uninstall() // Nothing to do here } - public async Task ExportAsync(Stream outStream, ExportImportOptions options, Action progressCallback, + public Task ExportAsync(Stream outStream, ExportImportOptions options, Action progressCallback, ICancellationToken cancellationToken) { - await _appBuilder.ApplicationServices.GetRequiredService().DoExportAsync(outStream, + return _appBuilder.ApplicationServices.GetRequiredService().DoExportAsync(outStream, progressCallback, cancellationToken); } - public async Task ImportAsync(Stream inputStream, ExportImportOptions options, Action progressCallback, + public Task ImportAsync(Stream inputStream, ExportImportOptions options, Action progressCallback, ICancellationToken cancellationToken) { - await _appBuilder.ApplicationServices.GetRequiredService().DoImportAsync(inputStream, + return _appBuilder.ApplicationServices.GetRequiredService().DoImportAsync(inputStream, progressCallback, cancellationToken); } } diff --git a/src/VirtoCommerce.SitemapsModule.Web/Scripts/blades/static-content-items-select.js b/src/VirtoCommerce.SitemapsModule.Web/Scripts/blades/static-content-items-select.js index a597a69..868f10b 100644 --- a/src/VirtoCommerce.SitemapsModule.Web/Scripts/blades/static-content-items-select.js +++ b/src/VirtoCommerce.SitemapsModule.Web/Scripts/blades/static-content-items-select.js @@ -23,17 +23,17 @@ function ($scope, $timeout, staticContent, bladeNavigationService, bladeUtils, u contentType: 'pages', storeId: blade.storeId, keyword: blade.searchKeyword, - folderUrl: blade.currentEntity.url + folderUrl: blade.currentEntity.relativeUrl }).$promise; var promises = [pagesPromise]; - if (!blade.currentEntity.url) { + if (!blade.currentEntity.relativeUrl) { var blogsPromise = staticContent.query({ contentType: 'blogs', storeId: blade.storeId, keyword: blade.searchKeyword, - folderUrl: blade.currentEntity.url + folderUrl: blade.currentEntity.relativeUrl }).$promise; promises.unshift(blogsPromise); diff --git a/src/VirtoCommerce.SitemapsModule.Web/module.manifest b/src/VirtoCommerce.SitemapsModule.Web/module.manifest index 98ae172..8f6db4d 100644 --- a/src/VirtoCommerce.SitemapsModule.Web/module.manifest +++ b/src/VirtoCommerce.SitemapsModule.Web/module.manifest @@ -1,7 +1,7 @@ VirtoCommerce.Sitemaps - 3.401.0 + 3.402.0 3.414.0