From 1eeffe4ae20adc7bd1cc52effcab5f69e9576165 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 29 Jan 2019 17:53:49 -0500 Subject: [PATCH] remove awx-manage migrate_to_database_settings --- .../commands/migrate_to_database_settings.py | 480 ------------------ awx/conf/settings.py | 2 +- awx/conf/views.py | 4 +- 3 files changed, 3 insertions(+), 483 deletions(-) delete mode 100644 awx/conf/management/commands/migrate_to_database_settings.py diff --git a/awx/conf/management/commands/migrate_to_database_settings.py b/awx/conf/management/commands/migrate_to_database_settings.py deleted file mode 100644 index eb6e0f8629..0000000000 --- a/awx/conf/management/commands/migrate_to_database_settings.py +++ /dev/null @@ -1,480 +0,0 @@ -# Copyright (c) 2016 Ansible, Inc. -# All Rights Reserved. - -# Python -import base64 -import collections -import difflib -import json -import os -import shutil - -# Django -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError -from django.db import transaction -from django.utils.text import slugify -from django.utils.timezone import now -from django.utils.translation import ugettext_lazy as _ - -# Tower -from awx import MODE -from awx.conf import settings_registry -from awx.conf.fields import empty, SkipField -from awx.conf.models import Setting -from awx.conf.utils import comment_assignments - - -class Command(BaseCommand): - - def add_arguments(self, parser): - parser.add_argument( - 'category', - nargs='*', - type=str, - ) - parser.add_argument( - '--dry-run', - action='store_true', - dest='dry_run', - default=False, - help=_('Only show which settings would be commented/migrated.'), - ) - parser.add_argument( - '--skip-errors', - action='store_true', - dest='skip_errors', - default=False, - help=_('Skip over settings that would raise an error when commenting/migrating.'), - ) - parser.add_argument( - '--no-comment', - action='store_true', - dest='no_comment', - default=False, - help=_('Skip commenting out settings in files.'), - ) - parser.add_argument( - '--comment-only', - action='store_true', - dest='comment_only', - default=False, - help=_('Skip migrating and only comment out settings in files.'), - ) - parser.add_argument( - '--backup-suffix', - dest='backup_suffix', - default=now().strftime('.%Y%m%d%H%M%S'), - help=_('Backup existing settings files with this suffix.'), - ) - - @transaction.atomic - def handle(self, *args, **options): - self.verbosity = int(options.get('verbosity', 1)) - self.dry_run = bool(options.get('dry_run', False)) - self.skip_errors = bool(options.get('skip_errors', False)) - self.no_comment = bool(options.get('no_comment', False)) - self.comment_only = bool(options.get('comment_only', False)) - self.backup_suffix = options.get('backup_suffix', '') - self.categories = options.get('category', None) or ['all'] - self.style.HEADING = self.style.MIGRATE_HEADING - self.style.LABEL = self.style.MIGRATE_LABEL - self.style.OK = self.style.SQL_FIELD - self.style.SKIP = self.style.WARNING - self.style.VALUE = self.style.SQL_KEYWORD - - # Determine if any categories provided are invalid. - category_slugs = [] - invalid_categories = [] - for category in self.categories: - category_slug = slugify(category) - if category_slug in settings_registry.get_registered_categories(): - if category_slug not in category_slugs: - category_slugs.append(category_slug) - else: - if category not in invalid_categories: - invalid_categories.append(category) - if len(invalid_categories) == 1: - raise CommandError('Invalid setting category: {}'.format(invalid_categories[0])) - elif len(invalid_categories) > 1: - raise CommandError('Invalid setting categories: {}'.format(', '.join(invalid_categories))) - - # Build a list of all settings to be migrated. - registered_settings = [] - for category_slug in category_slugs: - for registered_setting in settings_registry.get_registered_settings(category_slug=category_slug, read_only=False): - if registered_setting not in registered_settings: - registered_settings.append(registered_setting) - - self._migrate_settings(registered_settings) - - def _get_settings_file_patterns(self): - if MODE == 'development': - return [ - '/etc/tower/settings.py', - '/etc/tower/conf.d/*.py', - os.path.join(os.path.dirname(__file__), '..', '..', '..', 'settings', 'local_*.py') - ] - else: - return [ - os.environ.get('AWX_SETTINGS_FILE', '/etc/tower/settings.py'), - os.path.join(os.environ.get('AWX_SETTINGS_DIR', '/etc/tower/conf.d/'), '*.py'), - ] - - def _get_license_file(self): - return os.environ.get('AWX_LICENSE_FILE', '/etc/tower/license') - - def _comment_license_file(self, dry_run=True): - license_file = self._get_license_file() - diff_lines = [] - if os.path.exists(license_file): - try: - raw_license_data = open(license_file).read() - json.loads(raw_license_data) - except Exception as e: - raise CommandError('Error reading license from {0}: {1!r}'.format(license_file, e)) - if self.backup_suffix: - backup_license_file = '{}{}'.format(license_file, self.backup_suffix) - else: - backup_license_file = '{}.old'.format(license_file) - diff_lines = list(difflib.unified_diff( - raw_license_data.splitlines(), - [], - fromfile=backup_license_file, - tofile=license_file, - lineterm='', - )) - if not dry_run: - if self.backup_suffix: - shutil.copy2(license_file, backup_license_file) - os.remove(license_file) - return diff_lines - - def _get_local_settings_file(self): - if MODE == 'development': - static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static') - else: - static_root = settings.STATIC_ROOT - return os.path.join(static_root, 'local_settings.json') - - def _comment_local_settings_file(self, dry_run=True): - local_settings_file = self._get_local_settings_file() - diff_lines = [] - if os.path.exists(local_settings_file): - try: - raw_local_settings_data = open(local_settings_file).read() - json.loads(raw_local_settings_data) - except Exception as e: - if not self.skip_errors: - raise CommandError('Error reading local settings from {0}: {1!r}'.format(local_settings_file, e)) - return diff_lines - if self.backup_suffix: - backup_local_settings_file = '{}{}'.format(local_settings_file, self.backup_suffix) - else: - backup_local_settings_file = '{}.old'.format(local_settings_file) - diff_lines = list(difflib.unified_diff( - raw_local_settings_data.splitlines(), - [], - fromfile=backup_local_settings_file, - tofile=local_settings_file, - lineterm='', - )) - if not dry_run: - if self.backup_suffix: - shutil.copy2(local_settings_file, backup_local_settings_file) - os.remove(local_settings_file) - return diff_lines - - def _get_custom_logo_file(self): - if MODE == 'development': - static_root = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'ui', 'static') - else: - static_root = settings.STATIC_ROOT - return os.path.join(static_root, 'assets', 'custom_console_logo.png') - - def _comment_custom_logo_file(self, dry_run=True): - custom_logo_file = self._get_custom_logo_file() - diff_lines = [] - if os.path.exists(custom_logo_file): - try: - raw_custom_logo_data = open(custom_logo_file).read() - except Exception as e: - if not self.skip_errors: - raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e)) - return diff_lines - if self.backup_suffix: - backup_custom_logo_file = '{}{}'.format(custom_logo_file, self.backup_suffix) - else: - backup_custom_logo_file = '{}.old'.format(custom_logo_file) - diff_lines = list(difflib.unified_diff( - [''.format(len(raw_custom_logo_data))], - [], - fromfile=backup_custom_logo_file, - tofile=custom_logo_file, - lineterm='', - )) - if not dry_run: - if self.backup_suffix: - shutil.copy2(custom_logo_file, backup_custom_logo_file) - os.remove(custom_logo_file) - return diff_lines - - def _check_if_needs_comment(self, patterns, setting): - files_to_comment = [] - # If any diffs are returned, this setting needs to be commented. - diffs = comment_assignments(patterns, setting, dry_run=True) - if setting == 'LICENSE': - diffs.extend(self._comment_license_file(dry_run=True)) - elif setting == 'CUSTOM_LOGIN_INFO': - diffs.extend(self._comment_local_settings_file(dry_run=True)) - elif setting == 'CUSTOM_LOGO': - diffs.extend(self._comment_custom_logo_file(dry_run=True)) - for diff in diffs: - for line in diff.splitlines(): - if line.startswith('+++ '): - files_to_comment.append(line[4:]) - return files_to_comment - - def _check_if_needs_migration(self, setting): - # Check whether the current value differs from the default. - default_value = settings.DEFAULTS_SNAPSHOT.get(setting, empty) - if default_value is empty and setting != 'LICENSE': - field = settings_registry.get_setting_field(setting, read_only=True) - try: - default_value = field.get_default() - except SkipField: - pass - current_value = getattr(settings, setting, empty) - if setting == 'CUSTOM_LOGIN_INFO' and current_value in {empty, ''}: - local_settings_file = self._get_local_settings_file() - try: - if os.path.exists(local_settings_file): - local_settings = json.load(open(local_settings_file)) - current_value = local_settings.get('custom_login_info', '') - except Exception as e: - if not self.skip_errors: - raise CommandError('Error reading custom login info from {0}: {1!r}'.format(local_settings_file, e)) - if setting == 'CUSTOM_LOGO' and current_value in {empty, ''}: - custom_logo_file = self._get_custom_logo_file() - try: - if os.path.exists(custom_logo_file): - custom_logo_data = open(custom_logo_file).read() - if custom_logo_data: - current_value = 'data:image/png;base64,{}'.format(base64.b64encode(custom_logo_data)) - else: - current_value = '' - except Exception as e: - if not self.skip_errors: - raise CommandError('Error reading custom logo from {0}: {1!r}'.format(custom_logo_file, e)) - if current_value != default_value: - if current_value is empty: - current_value = None - return current_value - return empty - - def _display_tbd(self, setting, files_to_comment, migrate_value, comment_error=None, migrate_error=None): - if self.verbosity >= 1: - if files_to_comment: - if migrate_value is not empty: - action = 'Migrate + Comment' - else: - action = 'Comment' - if comment_error or migrate_error: - action = self.style.ERROR('{} (skipped)'.format(action)) - else: - action = self.style.OK(action) - self.stdout.write(' {}: {}'.format( - self.style.LABEL(setting), - action, - )) - if self.verbosity >= 2: - if migrate_error: - self.stdout.write(' - Migrate value: {}'.format( - self.style.ERROR(migrate_error), - )) - elif migrate_value is not empty: - self.stdout.write(' - Migrate value: {}'.format( - self.style.VALUE(repr(migrate_value)), - )) - if comment_error: - self.stdout.write(' - Comment: {}'.format( - self.style.ERROR(comment_error), - )) - elif files_to_comment: - for file_to_comment in files_to_comment: - self.stdout.write(' - Comment in: {}'.format( - self.style.VALUE(file_to_comment), - )) - else: - if self.verbosity >= 2: - self.stdout.write(' {}: {}'.format( - self.style.LABEL(setting), - self.style.SKIP('No Migration'), - )) - - def _display_migrate(self, setting, action, display_value): - if self.verbosity >= 1: - if action == 'No Change': - action = self.style.SKIP(action) - else: - action = self.style.OK(action) - self.stdout.write(' {}: {}'.format( - self.style.LABEL(setting), - action, - )) - if self.verbosity >= 2: - for line in display_value.splitlines(): - self.stdout.write(' {}'.format( - self.style.VALUE(line), - )) - - def _display_diff_summary(self, filename, added, removed): - self.stdout.write(' {} {}{} {}{}'.format( - self.style.LABEL(filename), - self.style.ERROR('-'), - self.style.ERROR(int(removed)), - self.style.OK('+'), - self.style.OK(str(added)), - )) - - def _display_comment(self, diffs): - for diff in diffs: - if self.verbosity >= 2: - for line in diff.splitlines(): - display_line = line - if line.startswith('--- ') or line.startswith('+++ '): - display_line = self.style.LABEL(line) - elif line.startswith('-'): - display_line = self.style.ERROR(line) - elif line.startswith('+'): - display_line = self.style.OK(line) - elif line.startswith('@@'): - display_line = self.style.VALUE(line) - if line.startswith('--- ') or line.startswith('+++ '): - self.stdout.write(' ' + display_line) - else: - self.stdout.write(' ' + display_line) - elif self.verbosity >= 1: - filename, lines_added, lines_removed = None, 0, 0 - for line in diff.splitlines(): - if line.startswith('+++ '): - if filename: - self._display_diff_summary(filename, lines_added, lines_removed) - filename, lines_added, lines_removed = line[4:], 0, 0 - elif line.startswith('+'): - lines_added += 1 - elif line.startswith('-'): - lines_removed += 1 - if filename: - self._display_diff_summary(filename, lines_added, lines_removed) - - def _discover_settings(self, registered_settings): - if self.verbosity >= 1: - self.stdout.write(self.style.HEADING('Discovering settings to be migrated and commented:')) - - # Determine which settings need to be commented/migrated. - to_migrate = collections.OrderedDict() - to_comment = collections.OrderedDict() - patterns = self._get_settings_file_patterns() - - for name in registered_settings: - comment_error, migrate_error = None, None - files_to_comment = [] - try: - files_to_comment = self._check_if_needs_comment(patterns, name) - except Exception as e: - comment_error = 'Error commenting {0}: {1!r}'.format(name, e) - if not self.skip_errors: - raise CommandError(comment_error) - if files_to_comment: - to_comment[name] = files_to_comment - migrate_value = empty - if files_to_comment: - migrate_value = self._check_if_needs_migration(name) - if migrate_value is not empty: - field = settings_registry.get_setting_field(name) - assert not field.read_only - try: - data = field.to_representation(migrate_value) - setting_value = field.run_validation(data) - db_value = field.to_representation(setting_value) - to_migrate[name] = db_value - except Exception as e: - to_comment.pop(name) - migrate_error = 'Unable to assign value {0!r} to setting "{1}: {2!s}".'.format(migrate_value, name, e) - if not self.skip_errors: - raise CommandError(migrate_error) - self._display_tbd(name, files_to_comment, migrate_value, comment_error, migrate_error) - if self.verbosity == 1 and not to_migrate and not to_comment: - self.stdout.write(' No settings found to migrate or comment!') - return (to_migrate, to_comment) - - def _migrate(self, to_migrate): - if self.verbosity >= 1: - if self.dry_run: - self.stdout.write(self.style.HEADING('Migrating settings to database (dry-run):')) - else: - self.stdout.write(self.style.HEADING('Migrating settings to database:')) - if not to_migrate: - self.stdout.write(' No settings to migrate!') - - # Now migrate those settings to the database. - for name, db_value in to_migrate.items(): - display_value = json.dumps(db_value, indent=4) - setting = Setting.objects.filter(key=name, user__isnull=True).order_by('pk').first() - action = 'No Change' - if not setting: - action = 'Migrated' - if not self.dry_run: - Setting.objects.create(key=name, user=None, value=db_value) - elif setting.value != db_value or type(setting.value) != type(db_value): - action = 'Updated' - if not self.dry_run: - setting.value = db_value - setting.save(update_fields=['value']) - self._display_migrate(name, action, display_value) - - def _comment(self, to_comment): - if self.verbosity >= 1: - if bool(self.dry_run or self.no_comment): - self.stdout.write(self.style.HEADING('Commenting settings in files (dry-run):')) - else: - self.stdout.write(self.style.HEADING('Commenting settings in files:')) - if not to_comment: - self.stdout.write(' No settings to comment!') - - # Now comment settings in settings files. - if to_comment: - to_comment_patterns = [] - license_file_to_comment = None - local_settings_file_to_comment = None - custom_logo_file_to_comment = None - for files_to_comment in to_comment.values(): - for file_to_comment in files_to_comment: - if file_to_comment == self._get_license_file(): - license_file_to_comment = file_to_comment - elif file_to_comment == self._get_local_settings_file(): - local_settings_file_to_comment = file_to_comment - elif file_to_comment == self._get_custom_logo_file(): - custom_logo_file_to_comment = file_to_comment - elif file_to_comment not in to_comment_patterns: - to_comment_patterns.append(file_to_comment) - # Run once in dry-run mode to catch any errors from updating the files. - diffs = comment_assignments(to_comment_patterns, list(to_comment.keys()), dry_run=True, backup_suffix=self.backup_suffix) - # Then, if really updating, run again. - if not self.dry_run and not self.no_comment: - diffs = comment_assignments(to_comment_patterns, list(to_comment.keys()), dry_run=False, backup_suffix=self.backup_suffix) - if license_file_to_comment: - diffs.extend(self._comment_license_file(dry_run=False)) - if local_settings_file_to_comment: - diffs.extend(self._comment_local_settings_file(dry_run=False)) - if custom_logo_file_to_comment: - diffs.extend(self._comment_custom_logo_file(dry_run=False)) - self._display_comment(diffs) - - def _migrate_settings(self, registered_settings): - to_migrate, to_comment = self._discover_settings(registered_settings) - - if not bool(self.comment_only): - self._migrate(to_migrate) - self._comment(to_comment) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 4719e3fe4a..0e05b97bee 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -299,7 +299,7 @@ class SettingsWrapper(UserSettingsHolder): self.__dict__['_awx_conf_preload_expires'] = time.time() + SETTING_CACHE_TIMEOUT # Check for any settings that have been defined in Python files and # make those read-only to avoid overriding in the database. - if not self._awx_conf_init_readonly and 'migrate_to_database_settings' not in sys.argv: + if not self._awx_conf_init_readonly: defaults_snapshot = self._get_default('DEFAULTS_SNAPSHOT') for key in get_writeable_settings(self.registry): init_default = defaults_snapshot.get(key, None) diff --git a/awx/conf/views.py b/awx/conf/views.py index 3e654f2add..d844fe3239 100644 --- a/awx/conf/views.py +++ b/awx/conf/views.py @@ -135,7 +135,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView): setting.value = value setting.save(update_fields=['value']) settings_change_list.append(key) - if settings_change_list and 'migrate_to_database_settings' not in sys.argv: + if settings_change_list: handle_setting_changes.delay(settings_change_list) def destroy(self, request, *args, **kwargs): @@ -150,7 +150,7 @@ class SettingSingletonDetail(RetrieveUpdateDestroyAPIView): continue setting.delete() settings_change_list.append(setting.key) - if settings_change_list and 'migrate_to_database_settings' not in sys.argv: + if settings_change_list: handle_setting_changes.delay(settings_change_list) # When TOWER_URL_BASE is deleted from the API, reset it to the hostname