diff --git a/wger/nutrition/management/commands/sync-ingredients.py b/wger/nutrition/management/commands/sync-ingredients.py index 32373fb52..431e1f3e0 100644 --- a/wger/nutrition/management/commands/sync-ingredients.py +++ b/wger/nutrition/management/commands/sync-ingredients.py @@ -50,11 +50,16 @@ class Command(WgerCommand): check_min_server_version(remote_url) try: - self.languages = languages - if self.languages is not None: - for language in self.languages.split(','): + if languages is not None: + for language in languages.split(','): validate_language_code(language) except ValidationError as e: raise CommandError('\n'.join([str(arg) for arg in e.args if arg is not None])) - sync_ingredients(self.stdout.write, self.remote_url, self.languages, self.style.SUCCESS) + sync_ingredients( + self.stdout.write, + self.remote_url, + languages, + self.style.SUCCESS, + show_progress_bar=True + ) diff --git a/wger/nutrition/sync.py b/wger/nutrition/sync.py index 2918dcff2..af06317ec 100644 --- a/wger/nutrition/sync.py +++ b/wger/nutrition/sync.py @@ -31,6 +31,7 @@ from openfoodfacts.images import ( AWS_S3_BASE_URL, generate_image_path, ) +from tqdm import tqdm # wger from wger.core.models.language import Language @@ -247,54 +248,71 @@ def sync_ingredients( remote_url=settings.WGER_SETTINGS['WGER_INSTANCE'], language_codes: Optional[str] = None, style_fn=lambda x: x, + show_progress_bar: bool = False, ): """Synchronize the ingredients from the remote server""" - - def _sync_ingredients(language_codes: Optional[int] = None): - if language_codes is not None: - url = make_uri( - INGREDIENTS_ENDPOINT, - server_url=remote_url, - query={'limit': API_MAX_ITEMS, 'language__in': language_codes}, - ) - else: - url = make_uri( - INGREDIENTS_ENDPOINT, - server_url=remote_url, - query={'limit': API_MAX_ITEMS}, - ) - for data in get_paginated(url, headers=wger_headers()): - uuid = data['uuid'] - name = data['name'] - - ingredient_data = extract_info_from_wger_api(data).dict() - ingredient_data['uuid'] = uuid - - ingredient, created = Ingredient.objects.update_or_create( - uuid=uuid, defaults=ingredient_data - ) - - print_fn(f'{"created" if created else "updated"} ingredient {uuid} - {name}') - print_fn('*** Synchronizing ingredients...') + language_ids: List[str] | None = None if language_codes is not None: - language_ids: List[int] = [] + language_ids = [] for code in language_codes.split(','): # Leaving the try except in here even though we've already validated on the sync-ingredients command itself. # This is in case we ever want to re-use this function for anything else where user can input language codes. try: lang = load_language(code, default_to_english=False) - language_ids.append(lang.id) + language_ids.append(str(lang.id)) except Language.DoesNotExist as e: print_fn( f'Error: The language code you provided ("{code}") does not exist in this database. Please try again.' ) return 0 - for language_id in language_ids: - _sync_ingredients(language_id) - else: - _sync_ingredients() + query: dict[str, str | int] = {'limit': API_MAX_ITEMS} + if language_ids is not None: + query['language__in'] = ','.join(language_ids) + + url = make_uri( + INGREDIENTS_ENDPOINT, + server_url=remote_url, + query=query, + ) + + # Fetch once to retrieve the number of results + response = requests.get(url, headers=wger_headers()).json() + total_ingredients = response['count'] + total_pages = total_ingredients // API_MAX_ITEMS + + ingredient_nr = 1 + page_nr = 1 + pbar = tqdm( + total=total_ingredients, + unit='ingredients', + desc='Syncing progress', + unit_scale=True + ) + for data in get_paginated(url, headers=wger_headers()): + uuid = data['uuid'] + + ingredient_data = extract_info_from_wger_api(data).dict() + ingredient_data['uuid'] = uuid + + Ingredient.objects.update_or_create( + uuid=uuid, + defaults=ingredient_data + ) + + if show_progress_bar: + pbar.update(1) + else: + # Note that get_paginated returns the individual result entries from the pages. + # To get the current page, we need to calculate this ourselves. + ingredient_nr += 1 + if ingredient_nr % API_MAX_ITEMS == 0: + page_nr += 1 + print_fn(f'Processing ingredients, page {page_nr: >4} of {total_pages}') + + pbar.close() print_fn(style_fn('done!\n')) + return None diff --git a/wger/nutrition/tasks.py b/wger/nutrition/tasks.py index a52418a06..0050e34f6 100644 --- a/wger/nutrition/tasks.py +++ b/wger/nutrition/tasks.py @@ -19,13 +19,12 @@ from random import ( randint, ) -# Django -from django.conf import settings -from django.core.management import call_command - # Third Party from celery import shared_task from celery.schedules import crontab +# Django +from django.conf import settings +from django.core.management import call_command # wger from wger.celery_configuration import app @@ -35,7 +34,6 @@ from wger.nutrition.sync import ( sync_ingredients, ) - logger = logging.getLogger(__name__) @@ -64,7 +62,10 @@ def sync_all_ingredients_task(): """ Fetches the current ingredients from the default wger instance """ - sync_ingredients(logger.info) + sync_ingredients( + logger.info, + show_progress_bar=False + ) @app.task