From b3344361cf40cdb2816098af9beca095cd17fdef Mon Sep 17 00:00:00 2001 From: tiago Date: Sat, 12 Oct 2024 16:35:18 +0100 Subject: [PATCH] feat(mtg): Add model MTGSets --- .github/workflows/test_backend.yml | 4 +- requirements.txt | 14 +- scripts/static_validate_ci_backend.sh | 4 +- src/cm_prices/settings.py | 1 + src/lib/utils.py | 258 ++++-------------- src/prices/admin.py | 17 +- src/prices/constants.py | 24 ++ .../migrations/0002_new_model_MTGSets.py | 65 +++++ src/prices/models.py | 31 ++- src/prices/services.py | 208 ++++++++++++++ 10 files changed, 414 insertions(+), 212 deletions(-) create mode 100644 src/prices/constants.py create mode 100644 src/prices/migrations/0002_new_model_MTGSets.py create mode 100644 src/prices/services.py diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index d5715cb..8b9ec97 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -26,10 +26,10 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python 3.12 + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.11 cache: 'pip' # - uses: actions/cache@v3 diff --git a/requirements.txt b/requirements.txt index d66836e..9346f93 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ -curl_cffi==0.7.2 +curl_cffi==0.7.3 requests==2.32.3 +bs4==0.0.2 Django==5.1.2 django-money==3.5.3 @@ -8,3 +9,14 @@ django-unfold==0.40.0 ptpython==3.0.29 pytz==2024.2 + +celery~=5.4.0 +semgrep~=1.90.0 +PyYAML~=6.0.2 + +# dev +black==24.10.0 +isort==5.13.2 +prospector==1.11.0 +bandit==1.7.10 +semgrep==1.90.0 \ No newline at end of file diff --git a/scripts/static_validate_ci_backend.sh b/scripts/static_validate_ci_backend.sh index ef44750..4974cff 100755 --- a/scripts/static_validate_ci_backend.sh +++ b/scripts/static_validate_ci_backend.sh @@ -3,7 +3,7 @@ export DJANGO_SETTINGS_MODULE=cm_prices.settings # exit on first non zero return status -set -e +set -x # ensure tee passes the error of the test tool # https://stackoverflow.com/questions/6871859/piping-command-output-to-tee-but-also-save-exit-code-of-command @@ -31,4 +31,4 @@ python manage.py makemigrations --check --dry-run bandit -r . # run python static validation -prospector --profile-path=. --profile=../.prospector.yml --path=. --ignore-patterns=static --output-format=xunit |& tee ../report_prospector.xml \ No newline at end of file +prospector --profile-path=. --profile=../.prospector.yml --path=. --ignore-patterns=static --output-format=xunit |& tee ../report_prospector.xml diff --git a/src/cm_prices/settings.py b/src/cm_prices/settings.py index 9c08301..27b27a9 100644 --- a/src/cm_prices/settings.py +++ b/src/cm_prices/settings.py @@ -128,6 +128,7 @@ # ('module.submodule2', 'function3'), # ('module.submodule3', '*'), # 'module.submodule4', + ('prices.services', '*'), ('lib.utils', '*'), ) diff --git a/src/lib/utils.py b/src/lib/utils.py index 2aa0e93..f66eb78 100644 --- a/src/lib/utils.py +++ b/src/lib/utils.py @@ -1,222 +1,78 @@ -import hashlib -import json import logging from datetime import datetime +from pathlib import Path import pytz -import requests -from prices.models import Catalog, MTGCard, MTGCardPrice +from prices.services import update_cm_prices logging.basicConfig(level=logging.INFO) # temporary logger = logging.getLogger(__name__) germany_tz = pytz.timezone('Europe/Berlin') -def update_mtg(): - """Fetch new cards, new prices and save them in the local models.""" +def simple_trend(price_dates, price_values): + """Calculate the trend (slope) of a card's price history using basic linear regression.""" - new_cards = update_cm_products() - updated_prices = update_cm_prices() - logger.info('Added new %d cards and updated %d prices', new_cards, updated_prices) + num_values = len(price_dates) + # Convert datetime objects to numeric values (e.g., days since the first date) + time_values = [(date - price_dates[0]).days for date in price_dates] -def update_cm_products(): - """Fetch and store product data for MTG cards.""" + # Calculate the sums + sum_time = sum(time_values) + sum_price = sum(price_values) + sum_time_price = sum(t * p for t, p in zip(time_values, price_values)) + sum_time_squared = sum(t**2 for t in time_values) - # Lists for bulk create/update - insert_cards = [] + # Calculate slope (trend) + numerator = num_values * sum_time_price - sum_time * sum_price + denominator = num_values * sum_time_squared - sum_time**2 + trend_slope = numerator / denominator if denominator != 0 else 0 - url = 'https://downloads.s3.cardmarket.com/productCatalog/productList/products_singles_1.json' + return trend_slope - response = requests.get(url, timeout=10) - if response.ok: - # skip catalog if already downloaded - md5sum = hashlib.md5(response.text.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep - if Catalog.objects.filter(md5sum=md5sum, catalog_type=Catalog.PRODUCTS).exists(): - catalog_date = Catalog.objects.get(md5sum=md5sum, catalog_type=Catalog.PRODUCTS).catalog_date - logger.info('Products already up to date since %s (%s)', catalog_date, md5sum) - return 0 - - data = response.json() - - catalog_date = data['createdAt'] - catalog_date = datetime.strptime(catalog_date, '%Y-%m-%dT%H:%M:%S%z') - Catalog.objects.create(catalog_date=catalog_date, md5sum=md5sum, catalog_type=Catalog.PRODUCTS) - - # List all existing cards within the current JSON - all_cm_ids = [item['idProduct'] for item in data['products']] - existing_cards = MTGCard.objects.filter(cm_id__in=all_cm_ids).in_bulk(field_name='cm_id') - - for product_item in data['products']: - cm_id = product_item['idProduct'] - name = product_item.get('name', None) - # slug = product_item.get('website', None).replace("/en/", "") if product_item.get('website') else None - - # Check if the card already exists - card = existing_cards.get(cm_id) - if not card: - - date_added = product_item.get('dateAdded') - if date_added: - date_added = datetime.strptime(date_added, '%Y-%m-%d %H:%M:%S') # convert str to date - date_added = date_added.replace(tzinfo=germany_tz) - - card = MTGCard( - cm_id=cm_id, - name=name, - category_id=product_item.get('idCategory', None), - expansion_id=product_item.get('idExpansion', None), - metacard_id=product_item.get('idMetacard', None), - cm_date_added=date_added, - ) - insert_cards.append(card) - # Bulk create new cards - if insert_cards: - MTGCard.objects.bulk_create(insert_cards) - logger.info('%d new cards inserted.', len(insert_cards)) +def trend(card, days=None): + """Find the trend of x days of a card pricing.""" - return len(insert_cards) - - -def update_cm_prices(): - """Fetch and store catalog prices for MTG cards.""" - - # Lists to be used in bulk_create - insert_prices = [] - - url = 'https://downloads.s3.cardmarket.com/productCatalog/priceGuide/price_guide_1.json' - - response = requests.get(url, timeout=10) - if response.ok: - # skip catalog if already downloaded - md5sum = hashlib.md5(response.text.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep - if Catalog.objects.filter(md5sum=md5sum, catalog_type=Catalog.PRICES).exists(): - catalog_date = Catalog.objects.get(md5sum=md5sum, catalog_type=Catalog.PRICES).catalog_date - logger.info('Prices already up to date since %s (%s)', catalog_date, md5sum) + if days: + days_ago = datetime.datetime.now() - datetime.timedelta(days=days) + if card.prices.filter(catalog_date__gte=days_ago).count() <= 1: + logger.warning('error getting trend from %d days', days_ago.days) return 0 - - data = response.json() - if data['version'] == 1: - - catalog_date = data['createdAt'] - catalog_date = datetime.strptime(catalog_date, '%Y-%m-%dT%H:%M:%S%z') - Catalog.objects.create(catalog_date=catalog_date, md5sum=md5sum, catalog_type=Catalog.PRICES) - - # List all existing cards within the current JSON - all_cm_ids = [item['idProduct'] for item in data['priceGuides']] - existing_cards = MTGCard.objects.filter(cm_id__in=all_cm_ids).in_bulk(field_name='cm_id') - - for price_item in data['priceGuides']: - cm_id = price_item['idProduct'] - - # Check if the card already exists - # products function should handle this - card = existing_cards.get(cm_id) - if not card: - # card = MTGCard(cm_id=cm_id) - # insert_cards.append(card) - logger.warning('Card with idProduct %s not found in MTGCard.', cm_id) - continue - - mtg_card_price = MTGCardPrice( - catalog_date=catalog_date, - card=card, - cm_id=cm_id, - avg=price_item.get('avg', None), - low=price_item.get('low', None), - trend=price_item.get('trend', None), - avg1=price_item.get('avg1', None), - avg7=price_item.get('avg7', None), - avg30=price_item.get('avg30', None), - avg_foil=price_item.get('avg-foil', None), - low_foil=price_item.get('low-foil', None), - trend_foil=price_item.get('trend-foil', None), - avg1_foil=price_item.get('avg1-foil', None), - avg7_foil=price_item.get('avg7-foil', None), - avg30_foil=price_item.get('avg30-foil', None), - ) - - insert_prices.append(mtg_card_price) - - # Bulk create all prices - if insert_prices: - MTGCardPrice.objects.bulk_create(insert_prices) - logger.info('%d new prices inserted.', len(insert_prices)) - - return len(insert_prices) - - -def save_local_prices(content): - """User previously saved jsons to save its prices info in the MTGCardPrice model.""" - - # from pathlib import Path - # directory = Path('../local/catalogs') - # for file in directory.glob("202*json"): - # with open(file, 'r') as f: - # content = f.read() - # try: - # save_local_prices(content) - # except Exception as err: - # print(err) - - # Lists to be used in bulk_create - insert_prices = [] - - # skip catalog if already downloaded - md5sum = hashlib.md5(content.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep - if Catalog.objects.filter(md5sum=md5sum, catalog_type=Catalog.PRICES).exists(): - catalog_date = Catalog.objects.get(md5sum=md5sum, catalog_type=Catalog.PRICES).catalog_date - logger.info('Prices already up to date since %s (%s)', catalog_date, md5sum) + prices = card.prices.filter(catalog_date__gte=days_ago) + else: + prices = card.prices.all() + prices = prices.exclude(trend__isnull=True) + if prices.count() <= 1: return 0 - - data = json.loads(content) - if data['version'] == 1: - - catalog_date = data['createdAt'] - catalog_date = datetime.strptime(catalog_date, '%Y-%m-%dT%H:%M:%S%z') - Catalog.objects.create(catalog_date=catalog_date, md5sum=md5sum, catalog_type=Catalog.PRICES) - - # List all existing cards within the current JSON - all_cm_ids = [item['idProduct'] for item in data['priceGuides']] - existing_cards = MTGCard.objects.filter(cm_id__in=all_cm_ids).in_bulk(field_name='cm_id') - - for price_item in data['priceGuides']: - cm_id = price_item['idProduct'] - - # Check if the card already exists - # products function should handle this - card = existing_cards.get(cm_id) - if not card: - # card = MTGCard(cm_id=cm_id) - # insert_cards.append(card) - logger.warning('Card with idProduct %d not found in MTGCard.', cm_id) - continue - - mtg_card_price = MTGCardPrice( - catalog_date=catalog_date, - card=card, - cm_id=cm_id, - avg=price_item.get('avg', None), - low=price_item.get('low', None), - trend=price_item.get('trend', None), - avg1=price_item.get('avg1', None), - avg7=price_item.get('avg7', None), - avg30=price_item.get('avg30', None), - avg_foil=price_item.get('avg-foil', None), - low_foil=price_item.get('low-foil', None), - trend_foil=price_item.get('trend-foil', None), - avg1_foil=price_item.get('avg1-foil', None), - avg7_foil=price_item.get('avg7-foil', None), - avg30_foil=price_item.get('avg30-foil', None), - ) - - insert_prices.append(mtg_card_price) - - # Bulk create all prices - if insert_prices: - MTGCardPrice.objects.bulk_create(insert_prices) - logger.info('%d new prices inserted.', len(insert_prices)) - - return len(insert_prices) + price_labels = list(prices.values_list('catalog_date', flat=True)) + price_values = list(prices.values_list('trend', flat=True)) + price_values = [0 if val is None else val for val in price_values] # replace None with 0 + + # price_labels = list(self.charted_prices.values_list('price_date', flat=True))[-days:] + # price_values = list(self.charted_prices.values_list('price_value', flat=True))[-days:] + + # x_values = np.linspace(0, 1, len(price_labels)) + # y_values = [float(x) for x in price_values] + # price_trend = np.polyfit(x_values, y_values, 1)[-2] + price_trend = simple_trend(price_labels, price_values) + + return price_trend + + +def update_from_local_files(): + """Update prices from local json files.""" + + directory = Path("../local/catalogs") + files = sorted(directory.glob("202*json"), key=lambda f: f.name) + for json in files: + # noset + with open(json, "r", encoding='utf-8') as file: + # content = json.load(f) + content = file.read() + try: + update_cm_prices(local_content=content) + except Exception as err: # NOQA + print(err) diff --git a/src/prices/admin.py b/src/prices/admin.py index 4185d36..9f9e05e 100644 --- a/src/prices/admin.py +++ b/src/prices/admin.py @@ -1,3 +1,16 @@ -# from django.contrib import admin +from django.contrib import admin +from unfold.admin import ModelAdmin -# Register your models here. +from prices.models import MTGCard +from prices.services import update_mtg + + +@admin.register(MTGCard) +class CustomAdminClass(ModelAdmin): + """Update all cardmarket prices from admin.""" + + actions = ['update_all'] + + def update_all(self): + """Update all cardmarket prices from admin.""" + update_mtg() diff --git a/src/prices/constants.py b/src/prices/constants.py new file mode 100644 index 0000000..f883463 --- /dev/null +++ b/src/prices/constants.py @@ -0,0 +1,24 @@ +LEGAL_STANDARD_SETS = [ + 5072, # Dominaria United + 5073, # Dominaria United: Extras + 5164, # The Brothers' War + 5165, # The Brothers' War: Extras + 5184, # Phyrexia: All Will Be One + 5191, # Phyrexia: All Will Be One: Extras + 5227, # March of the Machine + 5278, # March of the Machine: Extras + 5320, # March of the Machine: The Aftermath + 5208, # March of the Machine: The Aftermath: Extras + 5359, # Wilds of Eldraine + 5428, # Wilds of Eldraine: Extras + 5490, # The Lost Caverns of Ixalan + 5491, # The Lost Caverns of Ixalan: Extras + 5561, # Murders at Karlov Manor + 5599, # Murders at Karlov Manor: Extras + 5647, # Outlaws of Thunder Junction + 5662, # Outlaws of Thunder Junction: Extras + 5658, # Bloomburrow + 5659, # Bloomburrow: Extras + 5806, # Duskmourn: House of Horror + 5807, # Duskmourn: House of Horror: Extras +] diff --git a/src/prices/migrations/0002_new_model_MTGSets.py b/src/prices/migrations/0002_new_model_MTGSets.py new file mode 100644 index 0000000..d7aa20f --- /dev/null +++ b/src/prices/migrations/0002_new_model_MTGSets.py @@ -0,0 +1,65 @@ +# Generated by Django 5.1.2 on 2024-10-12 15:22 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("prices", "0001_initial_models"), + ] + + operations = [ + migrations.CreateModel( + name="MTGSet", + fields=[ + ( + "date_updated", + models.DateTimeField(auto_now=True, verbose_name="Last update at"), + ), + ( + "date_created", + models.DateTimeField(auto_now_add=True, verbose_name="Created at"), + ), + ("obs", models.TextField(blank=True, verbose_name="Observations")), + ("active", models.BooleanField(default=True, verbose_name="active")), + ( + "expansion_id", + models.PositiveSmallIntegerField( + primary_key=True, + serialize=False, + unique=True, + verbose_name="Cardmarket Set ID", + ), + ), + ( + "name", + models.CharField(max_length=255, unique=True, verbose_name="Set Name"), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.RemoveField( + model_name="mtgcard", + name="expansion_id", + ), + migrations.AlterField( + model_name="mtgcard", + name="name", + field=models.CharField(default="", max_length=255), + preserve_default=False, + ), + migrations.AddField( + model_name="mtgcard", + name="expansion", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="cards", + to="prices.mtgset", + ), + ), + ] diff --git a/src/prices/models.py b/src/prices/models.py index d058de8..870ec11 100644 --- a/src/prices/models.py +++ b/src/prices/models.py @@ -67,21 +67,44 @@ class Catalog(BaseAbstractModel): md5sum = models.CharField(max_length=32, verbose_name='MD5sum', unique=True) +class MTGSet(BaseAbstractModel): + """Model representing SET NAMES of MTG cards.""" + + expansion_id = models.PositiveSmallIntegerField(verbose_name='Cardmarket Set ID', unique=True, primary_key=True) + name = models.CharField(max_length=255, verbose_name='Set Name', unique=True) + + def __str__(self): + """Return representation in string format.""" + + return self.name + + class MTGCard(BaseAbstractModel): - """Model representing a MTG card.""" + """Model representing MTG card.""" cm_id = models.PositiveIntegerField(null=False, blank=False, primary_key=True) - name = models.CharField(max_length=255, null=True) - slug = models.SlugField(255, unique=True, null=True) # nosemgrep /en/Magic/Products/Singles/Mirage/Flash + name = models.CharField(max_length=255) + slug = models.SlugField(255, unique=True, null=True) # noset /en/Magic/Products/Singles/Mirage/Flash + + expansion = models.ForeignKey(MTGSet, on_delete=models.SET_NULL, null=True, related_name='cards') category_id = models.PositiveIntegerField(default=1) - expansion_id = models.PositiveIntegerField() + # expansion_id = models.PositiveIntegerField() metacard_id = models.PositiveIntegerField() cm_date_added = models.DateTimeField(verbose_name='Date added to cardmarket') class Meta: indexes = [models.Index(fields=['cm_id'], name='idx_mtgcard_cm_id')] + def __str__(self): + """Return representation in string format.""" + + latest_price = self.prices.order_by('-catalog_date').first() + trend_price = latest_price.trend if latest_price else 'No Price' + set_name = self.expansion.name if self.expansion else 'Unknown Set' + + return f"{self.name} - {set_name} - Trend: {trend_price}" + class MTGCardPrice(BaseAbstractModel): """MTG card price model.""" diff --git a/src/prices/services.py b/src/prices/services.py new file mode 100644 index 0000000..4342428 --- /dev/null +++ b/src/prices/services.py @@ -0,0 +1,208 @@ +import hashlib +import json +import logging +from datetime import datetime + +import pytz +import requests +from bs4 import BeautifulSoup +from curl_cffi import requests as curl + +from prices.models import Catalog, MTGCard, MTGCardPrice, MTGSet + +logging.basicConfig(level=logging.INFO) # temporary +logger = logging.getLogger(__name__) +germany_tz = pytz.timezone('Europe/Berlin') + + +def update_mtg(): + """Fetch new cards, new prices and save them in the local models.""" + + new_sets = update_cm_sets() + new_cards = update_cm_products() + updated_prices = update_cm_prices() + logger.info('Added new %d cards and updated %d prices', new_cards, updated_prices) + return new_sets, new_cards, updated_prices + + +def update_cm_products(): + """Fetch and store product data for MTG cards.""" + + # Lists for bulk create/update + insert_cards = [] + + url = 'https://downloads.s3.cardmarket.com/productCatalog/productList/products_singles_1.json' + + response = requests.get(url, timeout=10) + if response.ok: + # skip catalog if already downloaded + md5sum = hashlib.md5(response.text.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep + existing_catalog = Catalog.objects.filter(md5sum=md5sum, catalog_type=Catalog.PRODUCTS) + if existing_catalog.exists(): + catalog_date = existing_catalog.first().catalog_date + logger.info('Products already up to date since %s (%s)', catalog_date, md5sum) + return 0 + + data = response.json() + + catalog_date = data['createdAt'] + catalog_date = datetime.strptime(catalog_date, '%Y-%m-%dT%H:%M:%S%z') + Catalog.objects.create(catalog_date=catalog_date, md5sum=md5sum, catalog_type=Catalog.PRODUCTS) + + # List all existing cards within the current JSON + all_cm_ids = [item['idProduct'] for item in data['products']] + existing_cards = MTGCard.objects.filter(cm_id__in=all_cm_ids).in_bulk(field_name='cm_id') + + for product_item in data['products']: + cm_id = product_item['idProduct'] + name = product_item.get('name', None) + # slug = product_item.get('website', None).replace("/en/", "") if product_item.get('website') else None + + # Check if the card already exists + card = existing_cards.get(cm_id) + if not card: + + date_added = product_item.get('dateAdded') + if date_added: + date_added = datetime.strptime(date_added, '%Y-%m-%d %H:%M:%S') # convert str to date + date_added = date_added.replace(tzinfo=germany_tz) + + card = MTGCard( + cm_id=cm_id, + name=name, + category_id=product_item.get('idCategory', None), + expansion_id=product_item.get('idExpansion', None), + metacard_id=product_item.get('idMetacard', None), + cm_date_added=date_added, + ) + insert_cards.append(card) + + # Bulk create new cards + if insert_cards: + MTGCard.objects.bulk_create(insert_cards) + logger.info('%d new cards inserted.', len(insert_cards)) + + return len(insert_cards) + + +def update_cm_prices(local_content=None): + """Fetch and store catalog prices for MTG cards.""" + + # Lists to be used in bulk_create + insert_prices = [] + + # ############ previously downloaded json files + + # from pathlib import Path + # + # directory = Path("../local/catalogs") + # files = sorted(directory.glob("202*json"), key=lambda f: f.name) + # for file in files: + # with open(file, "r") as f: + # # content = json.load(f) + # content = f.read() + # try: + # update_cm_prices(local_content=content) + # except Exception as err: + # print(err) + if local_content: + md5sum = hashlib.md5(local_content.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep + try: + content = json.loads(local_content) + except json.JSONDecodeError as e: + logger.error("Failed to decode JSON: %s", e) + return 0 + + # ############ Typical behaviour + else: + url = 'https://downloads.s3.cardmarket.com/productCatalog/priceGuide/price_guide_1.json' + response = requests.get(url, timeout=10) + if response.ok: + content = response.json() + md5sum = hashlib.md5(response.text.encode('utf-8'), usedforsecurity=False).hexdigest() # nosemgrep + else: + logger.error('Unable to download JSON: %s', response.text) + return 0 + + existing_catalog = Catalog.objects.filter(md5sum=md5sum, catalog_type=Catalog.PRICES) + if existing_catalog.exists(): + catalog_date = existing_catalog.first().catalog_date + logger.info('Prices already up to date since %s (%s)', catalog_date, md5sum) + return 0 + + data = content + if data['version'] == 1: + + catalog_date = data['createdAt'] + catalog_date = datetime.strptime(catalog_date, '%Y-%m-%dT%H:%M:%S%z') + Catalog.objects.create(catalog_date=catalog_date, md5sum=md5sum, catalog_type=Catalog.PRICES) + + # List all existing cards within the current JSON + all_cm_ids = [item['idProduct'] for item in data['priceGuides']] + existing_cards = MTGCard.objects.filter(cm_id__in=all_cm_ids).in_bulk(field_name='cm_id') + + for price_item in data['priceGuides']: + cm_id = price_item['idProduct'] + + # Check if the card already exists + # products function should handle this + card = existing_cards.get(cm_id) + if not card: + # card = MTGCard(cm_id=cm_id) + # insert_cards.append(card) + logger.warning('Card with idProduct %s not found in MTGCard.', cm_id) + continue + + mtg_card_price = MTGCardPrice( + catalog_date=catalog_date, + card=card, + cm_id=cm_id, + avg=price_item.get('avg', None), + low=price_item.get('low', None), + trend=price_item.get('trend', None), + avg1=price_item.get('avg1', None), + avg7=price_item.get('avg7', None), + avg30=price_item.get('avg30', None), + avg_foil=price_item.get('avg-foil', None), + low_foil=price_item.get('low-foil', None), + trend_foil=price_item.get('trend-foil', None), + avg1_foil=price_item.get('avg1-foil', None), + avg7_foil=price_item.get('avg7-foil', None), + avg30_foil=price_item.get('avg30-foil', None), + ) + + insert_prices.append(mtg_card_price) + + # Bulk create all prices + if insert_prices: + MTGCardPrice.objects.bulk_create(insert_prices) + logger.info('%d new prices inserted.', len(insert_prices)) + + return len(insert_prices) + + +def update_cm_sets(): + """Update cardmarket set names and ids.""" + + url = "https://www.cardmarket.com/en/Magic/Products/Singles" + created_sets = 0 + + response = curl.get(url, impersonate='chrome') + if response.ok: + soup = BeautifulSoup(response.text, 'html.parser') + cm_sets = soup.find('select', attrs={'name': 'idExpansion'}) + existing_sets = MTGSet.objects.values_list('expansion_id', flat=True) + + for opt in cm_sets.find_all('option'): + set_id = int(opt.get('value').strip()) + if set_id == 0: # skip "All" option + continue + + if set_id not in existing_sets: + set_name = opt.text.strip() + new_set = MTGSet(name=set_name, expansion_id=set_id) + new_set.save() + created_sets += 1 + logger.info('Created new set %s.', set_name) + + return created_sets