Skip to content

Commit

Permalink
Merge branch 'release/3.402.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@vc-ci
vc-ci committed Dec 11, 2023
2 parents 0196064 + 9ba6f5f commit 882e303
Show file tree
Hide file tree
Showing 10 changed files with 110 additions and 77 deletions.
2 changes: 1 addition & 1 deletion Directory.Build.props
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<Project>
<!-- These properties will be shared for all projects -->
<PropertyGroup>
<VersionPrefix>3.401.0</VersionPrefix>
<VersionPrefix>3.402.0</VersionPrefix>
<VersionSuffix></VersionSuffix>
<VersionSuffix Condition=" '$(VersionSuffix)' != '' AND '$(BuildNumber)' != '' ">$(VersionSuffix)-$(BuildNumber)</VersionSuffix>
</PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion src/VirtoCommerce.SitemapsModule.Data/Services/ISitemapXmlGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace VirtoCommerce.SitemapsModule.Data.Services
{
public interface ISitemapXmlGenerator
{
Task<ICollection<string>> GetSitemapUrlsAsync(string storeId);
Task<ICollection<string>> GetSitemapUrlsAsync(string storeId, string baseUrl);

Task<Stream> GenerateSitemapXmlAsync(string storeId, string baseUrl, string sitemapUrl, Action<ExportImportProgressInfo> progressCallback = null);
}
Expand Down
16 changes: 11 additions & 5 deletions ...temapsModule.Data/Services/SitemapItemRecordProviders/CatalogSitemapItemRecordProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ protected virtual async Task LoadCategoriesSitemapItemRecordsAsync(Store store,
{
// If images need to be included - run a search for picked products to get variations with images
var productIds = result.Results.Where(x => x is ProductListEntry).Select(x => x.Id).ToArray();
products = await SearchProductsWithVariations(batchSize, productIds);
products = await SearchProductsWithVariations(productIds);
}

foreach (var listEntry in result.Results)
Expand Down Expand Up @@ -147,7 +147,7 @@ protected virtual async Task LoadProductsSitemapItemRecordsAsync(Store store, Si
}
}

private async Task<List<CatalogProduct>> SearchProductsWithVariations(int batchSize, string[] productIds = null, ProductSearchCriteria searchCriteria = null, Action<ExportImportProgressInfo> progressCallback = null)
private async Task<List<CatalogProduct>> SearchProductsWithVariations(string[] productIds = null)
{
var products = (await _itemService.GetAsync(productIds, (ItemResponseGroup.Seo | ItemResponseGroup.Outlines | ItemResponseGroup.WithImages).ToString()))
.Where(p => !p.IsActive.HasValue || p.IsActive.Value).ToList();
Expand All @@ -166,15 +166,17 @@ private async Task<List<CatalogProduct>> SearchProductsWithVariations(int batchS
/// <returns></returns>
private async Task LoadProductsWithImages(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
{
var batchSize = await _settingsManager.GetValueAsync<int>(ModuleConstants.Settings.General.SearchBunchSize);

var productSitemapItems = sitemap.Items.Where(x => x.ObjectType.EqualsInvariant(SitemapItemTypes.Product)).ToList();

var productOptions = GetProductOptions(store);

var productIds = productSitemapItems.Select(x => x.ObjectId).ToArray();

var products = await SearchProductsWithVariations(batchSize, productIds);
var products = await SearchProductsWithVariations(productIds);

var progressInfo = new ExportImportProgressInfo();

var count = 0;

foreach (var product in products)
{
Expand All @@ -197,6 +199,10 @@ private async Task LoadProductsWithImages(Store store, Sitemap sitemap, string b

productSitemapItem.ItemsRecords.AddRange(itemRecords);
}

count++;
progressInfo.Description = $"Catalog: Have been generated {count} of {products.Count} records for products items";
progressCallback?.Invoke(progressInfo);
}
}

Expand Down
117 changes: 71 additions & 46 deletions ...Module.Data/Services/SitemapItemRecordProviders/StaticContentSitemapItemRecordProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using VirtoCommerce.AssetsModule.Core.Assets;
using VirtoCommerce.ContentModule.Core.Model;
using VirtoCommerce.ContentModule.Core.Services;
using VirtoCommerce.Platform.Core.Common;
using VirtoCommerce.Platform.Core.ExportImport;
Expand All @@ -13,95 +14,81 @@
using VirtoCommerce.SitemapsModule.Core.Models;
using VirtoCommerce.SitemapsModule.Core.Services;
using VirtoCommerce.SitemapsModule.Data.Extensions;
using VirtoCommerce.StoreModule.Core.Model;
using VirtoCommerce.Tools;
using YamlDotNet.RepresentationModel;
using Store = VirtoCommerce.StoreModule.Core.Model.Store;

namespace VirtoCommerce.SitemapsModule.Data.Services.SitemapItemRecordProviders
{
public class StaticContentSitemapItemRecordProvider : SitemapItemRecordProviderBase, ISitemapItemRecordProvider
{
private readonly ISettingsManager _settingsManager;
private readonly IBlobContentStorageProviderFactory _blobStorageProviderFactory;
private const string PagesContentType = "pages";
private static readonly Regex _headerRegExp = new Regex(@"(?s:^---(.*?)---)");

private readonly ISettingsManager _settingsManager;
private readonly IContentFileService _contentFileService;
private readonly IContentService _contentService;

public StaticContentSitemapItemRecordProvider(
ISitemapUrlBuilder urlBuilder,
ISettingsManager settingsManager,
IBlobContentStorageProviderFactory blobStorageProviderFactory)
IContentService contentService,
IContentFileService contentFileService)
: base(urlBuilder)
{
_settingsManager = settingsManager;
_blobStorageProviderFactory = blobStorageProviderFactory;
_contentService = contentService;
_contentFileService = contentFileService;
}

public virtual async Task LoadSitemapItemRecordsAsync(Store store, Sitemap sitemap, string baseUrl, Action<ExportImportProgressInfo> progressCallback = null)
{
var progressInfo = new ExportImportProgressInfo();

var contentBasePath = $"Pages/{sitemap.StoreId}";
var storageProvider = _blobStorageProviderFactory.CreateProvider(contentBasePath);

var staticContentSitemapItems = sitemap.Items
.Where(si => !string.IsNullOrEmpty(si.ObjectType))
.Where(si => si.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) || si.ObjectType.EqualsInvariant(SitemapItemTypes.Folder))
.ToList();
var staticContentSitemapItems = GetStaticContentSitemapItems(sitemap);

var totalCount = staticContentSitemapItems.Count;
if (totalCount <= 0)
if (totalCount == 0)
{
return;
}

var processedCount = 0;

var acceptedFilenameExtensions = (await _settingsManager.GetValueAsync<string>(ModuleConstants.Settings.General.AcceptedFilenameExtensions))
.Split(',')
.Select(i => i.Trim())
.Where(i => !string.IsNullOrEmpty(i))
.ToList();
var allowedExtensions = await GetAllowedExtensions();
var blogOptions = await GetBlogOptions(store);

progressInfo.Description = $"Content: Starting records generation for {totalCount} pages";
progressCallback?.Invoke(progressInfo);

foreach (var sitemapItem in staticContentSitemapItems)
{
var urls = new List<string>();
var validSitemapItems = new List<string>();

if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.Folder))
{
var searchResult = await storageProvider.SearchAsync(sitemapItem.UrlTemplate, null);

if (searchResult.TotalCount == 0)
{
searchResult = await storageProvider.SearchAsync("blogs/" + sitemapItem.UrlTemplate, null);
}

var itemUrls = await GetItemUrls(storageProvider, searchResult);
foreach (var itemUrl in itemUrls.Where(itemUrl => IsExtensionAllowed(acceptedFilenameExtensions, itemUrl)))
{
urls.Add(itemUrl);
}
await LoadPagesRecursivly(sitemap.StoreId, sitemapItem.UrlTemplate, allowedExtensions, validSitemapItems);
}
else if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem))
else if (sitemapItem.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) &&
IsExtensionAllowed(allowedExtensions, sitemapItem.UrlTemplate) &&
await _contentService.ItemExistsAsync(PagesContentType, sitemap.StoreId, sitemapItem.UrlTemplate))
{
var item = await storageProvider.GetBlobInfoAsync(sitemapItem.UrlTemplate);
if (item != null && IsExtensionAllowed(acceptedFilenameExtensions, item.RelativeUrl))
{
urls.Add(item.RelativeUrl);
}
validSitemapItems.Add(sitemapItem.UrlTemplate);
}

totalCount = urls.Count;
totalCount = validSitemapItems.Count;

foreach (var url in urls)
foreach (var url in validSitemapItems)
{
using (var stream = await storageProvider.OpenReadAsync(url))
var contentFile = await _contentService.GetFileAsync(PagesContentType, sitemap.StoreId, url);

using (var stream = await _contentService.GetItemStreamAsync(PagesContentType, sitemap.StoreId, url))
{
var content = stream.ReadToString();
var frontMatterPermalink = GetPermalink(content, url);
frontMatterPermalink.FilePath = url;

var frontMatterPermalink = GetPermalink(content, url, contentFile.Url);
var urlTemplate = frontMatterPermalink.ToUrl().TrimStart('/');
var blogOptions = await GetBlogOptions(store);

var records = GetSitemapItemRecords(store, blogOptions, urlTemplate, baseUrl);
sitemapItem.ItemsRecords.AddRange(records);
}
Expand All @@ -113,6 +100,43 @@ public virtual async Task LoadSitemapItemRecordsAsync(Store store, Sitemap sitem
}
}

private async Task LoadPagesRecursivly(string storeId, string folrderUrl, List<string> allowedExtensions, List<string> validSitemapItems)
{
var criteria = AbstractTypeFactory<FilterItemsCriteria>.TryCreateInstance();
criteria.ContentType = PagesContentType;
criteria.StoreId = storeId;
criteria.FolderUrl = folrderUrl;

var searchResult = await _contentFileService.FilterItemsAsync(criteria);

foreach (var file in searchResult.Where(file => file.Type == "blob" && IsExtensionAllowed(allowedExtensions, file.RelativeUrl)))
{
validSitemapItems.Add(file.RelativeUrl);
}

// Load Pages from SubFolders
foreach (var folder in searchResult.Where(file => file.Type == "folder"))
{
await LoadPagesRecursivly(storeId, folder.RelativeUrl, allowedExtensions, validSitemapItems);
}
}

private static List<SitemapItem> GetStaticContentSitemapItems(Sitemap sitemap)
{
return sitemap.Items
.Where(si => !string.IsNullOrEmpty(si.ObjectType))
.Where(si => si.ObjectType.EqualsInvariant(SitemapItemTypes.ContentItem) || si.ObjectType.EqualsInvariant(SitemapItemTypes.Folder))
.ToList();
}

private async Task<List<string>> GetAllowedExtensions()
{
return (await _settingsManager.GetValueAsync<string>(ModuleConstants.Settings.General.AcceptedFilenameExtensions))
.Split(',')
.Select(i => i.Trim())
.Where(i => !string.IsNullOrEmpty(i))
.ToList();
}

private async Task<SitemapItemOptions> GetBlogOptions(Store store)
{
Expand Down Expand Up @@ -142,7 +166,7 @@ private static bool IsExtensionAllowed(IList<string> acceptedFilenameExtensions,
return string.IsNullOrEmpty(itemExtension) || acceptedFilenameExtensions.Contains(itemExtension, StringComparer.OrdinalIgnoreCase);
}

private static FrontMatterPermalink GetPermalink(string content, string url)
private static FrontMatterPermalink GetPermalink(string content, string url, string filePath)
{
if (content.TryParseJson(out var token) && token.HasValues && token.First?["permalink"] != null)
{
Expand All @@ -153,10 +177,10 @@ private static FrontMatterPermalink GetPermalink(string content, string url)
yamlHeader.TryGetValue("permalink", out var permalinks);
if (permalinks != null)
{
return new FrontMatterPermalink(permalinks.FirstOrDefault());
return new FrontMatterPermalink(permalinks.FirstOrDefault()) { FilePath = filePath };
}

return new FrontMatterPermalink(url.Replace(".md", ""));
return new FrontMatterPermalink(url.Replace(".md", "")) { FilePath = filePath };
}

private static async Task<ICollection<string>> GetItemUrls(IBlobContentStorageProvider storageProvider, GenericSearchResult<BlobEntry> searchResult)
Expand Down Expand Up @@ -224,3 +248,4 @@ private static IEnumerable<string> GetYamlNodeValues(YamlNode value)
}
}
}
;
4 changes: 4 additions & 0 deletions src/VirtoCommerce.SitemapsModule.Data/Services/SitemapUrlBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,10 @@ public SitemapUrlBuilder(IUrlBuilder urlBuilder)
public virtual string BuildStoreUrl(Store store, string language, string urlTemplate, string baseUrl, IEntity entity = null)
{
var toolsStore = store.ToToolsStore(baseUrl);
if (!string.IsNullOrEmpty(baseUrl))
{
toolsStore.Url = baseUrl;
}

var seoSupport = entity as ISeoSupport;

Expand Down
18 changes: 8 additions & 10 deletions src/VirtoCommerce.SitemapsModule.Data/Services/SitemapXmlGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ public SitemapXmlGenerator(
_storeService = storeService;
}

public virtual async Task<ICollection<string>> GetSitemapUrlsAsync(string storeId)
public virtual async Task<ICollection<string>> GetSitemapUrlsAsync(string storeId, string baseUrl)
{
if (string.IsNullOrEmpty(storeId))
{
Expand All @@ -56,7 +56,7 @@ public virtual async Task<ICollection<string>> GetSitemapUrlsAsync(string storeI
var sitemapUrls = new List<string>();
var store = await _storeService.GetByIdAsync(storeId, StoreResponseGroup.StoreInfo.ToString());

var sitemaps = await LoadAllStoreSitemaps(store, "");
var sitemaps = await LoadStoreSitemaps(store, baseUrl);
foreach (var sitemap in sitemaps)
{
sitemapUrls.AddRange(sitemap.PagedLocations);
Expand All @@ -73,7 +73,6 @@ public virtual async Task<Stream> GenerateSitemapXmlAsync(string storeId, string
var recordsLimitPerFile = await _settingsManager.GetValueAsync<int>(ModuleConstants.Settings.General.RecordsLimitPerFile);

var xmlNamespaces = new XmlSerializerNamespaces();
//xmlNamespaces.Add("", "https://www.sitemaps.org/schemas/sitemap/0.9");
xmlNamespaces.Add("xhtml", "https://www.w3.org/1999/xhtml");
xmlNamespaces.Add("image", "http://www.google.com/schemas/sitemap-image/1.1");

Expand All @@ -86,15 +85,14 @@ public virtual async Task<Stream> GenerateSitemapXmlAsync(string storeId, string
Description = "Creating sitemap.xml..."
});

var allStoreSitemaps = await LoadAllStoreSitemaps(store, baseUrl);
var storeSitemaps = await LoadStoreSitemaps(store, baseUrl);

var sitemapIndexXmlRecord = new SitemapIndexXmlRecord();

foreach (var sitemap in allStoreSitemaps)
foreach (var sitemap in storeSitemaps)
{
var xmlSiteMapRecords = sitemap.PagedLocations.Select(location => new SitemapIndexItemXmlRecord
{
//ModifiedDate = sitemap.Items.Select(x => x.ModifiedDate).OrderByDescending(x => x).FirstOrDefault()?.ToString("yyyy-MM-dd"),
Url = _sitemapUrlBuilder.BuildStoreUrl(store, store.DefaultLanguage, location, baseUrl),
}).ToList();

Expand All @@ -112,7 +110,7 @@ public virtual async Task<Stream> GenerateSitemapXmlAsync(string storeId, string
{
await LoadSitemapRecords(store, sitemap, baseUrl, progressCallback);

var distinctRecords = sitemap.Items.SelectMany(x => x.ItemsRecords);//.GroupBy(x => x.Url).Select(x => x.FirstOrDefault());
var distinctRecords = sitemap.Items.SelectMany(x => x.ItemsRecords);
var sitemapItemRecords = distinctRecords.Skip((sitemapLocation.PageNumber - 1) * recordsLimitPerFile).Take(recordsLimitPerFile).ToArray();

var sitemapRecord = new SitemapXmlRecord
Expand All @@ -132,17 +130,18 @@ public virtual async Task<Stream> GenerateSitemapXmlAsync(string storeId, string
return stream;
}

private async Task<ICollection<Sitemap>> LoadAllStoreSitemaps(Store store, string baseUrl)
private async Task<ICollection<Sitemap>> LoadStoreSitemaps(Store store, string baseUrl)
{
var sitemaps = new List<Sitemap>();
var sitemapSearchCriteria = new SitemapSearchCriteria
{
StoreId = store.Id,
Skip = 0,
Take = int.MaxValue
};

var sitemapSearchResult = await _sitemapSearchService.SearchAsync(sitemapSearchCriteria);

var sitemaps = new List<Sitemap>();
foreach (var sitemap in sitemapSearchResult.Results)
{
await LoadSitemapRecords(store, sitemap, baseUrl);
Expand All @@ -166,7 +165,6 @@ private async Task LoadSitemapRecords(Store store, Sitemap sitemap, string baseU
};
sitemap.Items = (await _sitemapItemSearchService.SearchAsync(sitemapItemSearchCriteria)).Results;

var imageUrls = new List<SitemapItemImageRecord>();
foreach (var recordProvider in _sitemapItemRecordProviders)
{
//Log exceptions to prevent fail whole sitemap.xml generation
Expand Down
Loading

0 comments on commit 882e303

Please sign in to comment.