from __future__ import unicode_literals import copy from collections import OrderedDict from django.apps import AppConfig from django.apps.registry import Apps, apps as global_apps from django.conf import settings from django.db import models from django.db.models.fields.proxy import OrderWrt from django.db.models.fields.related import ( RECURSIVE_RELATIONSHIP_CONSTANT, do_pending_lookups, ) from django.db.models.options import DEFAULT_NAMES, normalize_together from django.utils import six from django.utils.encoding import force_text, smart_text from django.utils.functional import cached_property from django.utils.module_loading import import_string from django.utils.version import get_docs_version class InvalidBasesError(ValueError): pass def _get_app_label_and_model_name(model, app_label=''): if isinstance(model, six.string_types): split = model.split('.', 1) return (tuple(split) if len(split) == 2 else (app_label, split[0])) else: return model._meta.app_label, model._meta.model_name def get_related_models_recursive(model): """ Returns all models that have a direct or indirect relationship to the given model. """ def _related_models(m): return [ f.related_model for f in m._meta.get_fields(include_parents=True, include_hidden=True) if f.is_relation and not isinstance(f.related_model, six.string_types) ] + [ subclass for subclass in m.__subclasses__() if issubclass(subclass, models.Model) ] seen = set() queue = _related_models(model) for rel_mod in queue: rel_app_label, rel_model_name = rel_mod._meta.app_label, rel_mod._meta.model_name if (rel_app_label, rel_model_name) in seen: continue seen.add((rel_app_label, rel_model_name)) queue.extend(_related_models(rel_mod)) return seen - {(model._meta.app_label, model._meta.model_name)} class ProjectState(object): """ Represents the entire project's overall state. This is the item that is passed around - we do it here rather than at the app level so that cross-app FKs/etc. resolve properly. """ def __init__(self, models=None, real_apps=None): self.models = models or {} # Apps to include from main registry, usually unmigrated ones self.real_apps = real_apps or [] def add_model(self, model_state): app_label, model_name = model_state.app_label, model_state.name_lower self.models[(app_label, model_name)] = model_state if 'apps' in self.__dict__: # hasattr would cache the property self.reload_model(app_label, model_name) def remove_model(self, app_label, model_name): del self.models[app_label, model_name] if 'apps' in self.__dict__: # hasattr would cache the property self.apps.unregister_model(app_label, model_name) def reload_model(self, app_label, model_name): if 'apps' in self.__dict__: # hasattr would cache the property try: old_model = self.apps.get_model(app_label, model_name) except LookupError: related_models = set() else: # Get all relations to and from the old model before reloading, # as _meta.apps may change related_models = get_related_models_recursive(old_model) # Get all outgoing references from the model to be rendered model_state = self.models[(app_label, model_name)] for name, field in model_state.fields: if field.is_relation: if field.rel.to == RECURSIVE_RELATIONSHIP_CONSTANT: continue rel_app_label, rel_model_name = _get_app_label_and_model_name(field.rel.to, app_label) related_models.add((rel_app_label, rel_model_name.lower())) # Unregister all related models for rel_app_label, rel_model_name in related_models: self.apps.unregister_model(rel_app_label, rel_model_name) # Unregister the current model self.apps.unregister_model(app_label, model_name) # Gather all models states of those models that will be rerendered. # This includes: # 1. The current model try: model_state = self.models[app_label, model_name] except KeyError: states_to_be_rendered = [] else: states_to_be_rendered = [model_state] # 2. All related models of unmigrated apps for model_state in self.apps.real_models: if (model_state.app_label, model_state.name_lower) in related_models: states_to_be_rendered.append(model_state) # 3. All related models of migrated apps for rel_app_label, rel_model_name in related_models: try: model_state = self.models[rel_app_label, rel_model_name] except KeyError: pass else: states_to_be_rendered.append(model_state) # Render all models self.apps.render_multiple(states_to_be_rendered) def clone(self): "Returns an exact copy of this ProjectState" new_state = ProjectState( models={k: v.clone() for k, v in self.models.items()}, real_apps=self.real_apps, ) if 'apps' in self.__dict__: new_state.apps = self.apps.clone() return new_state @cached_property def apps(self): return StateApps(self.real_apps, self.models) @property def concrete_apps(self): self.apps = StateApps(self.real_apps, self.models, ignore_swappable=True) return self.apps @classmethod def from_apps(cls, apps): "Takes in an Apps and returns a ProjectState matching it" app_models = {} for model in apps.get_models(include_swapped=True): model_state = ModelState.from_model(model) app_models[(model_state.app_label, model_state.name_lower)] = model_state return cls(app_models) def __eq__(self, other): if set(self.models.keys()) != set(other.models.keys()): return False if set(self.real_apps) != set(other.real_apps): return False return all(model == other.models[key] for key, model in self.models.items()) def __ne__(self, other): return not (self == other) class AppConfigStub(AppConfig): """ Stubs a Django AppConfig. Only provides a label, and a dict of models. """ # Not used, but required by AppConfig.__init__ path = '' def __init__(self, label): self.label = label # App-label and app-name are not the same thing, so technically passing # in the label here is wrong. In practice, migrations don't care about # the app name, but we need something unique, and the label works fine. super(AppConfigStub, self).__init__(label, None) def import_models(self, all_models): self.models = all_models class StateApps(Apps): """ Subclass of the global Apps registry class to better handle dynamic model additions and removals. """ def __init__(self, real_apps, models, ignore_swappable=False): # Any apps in self.real_apps should have all their models included # in the render. We don't use the original model instances as there # are some variables that refer to the Apps object. # FKs/M2Ms from real apps are also not included as they just # mess things up with partial states (due to lack of dependencies) self.real_models = [] for app_label in real_apps: app = global_apps.get_app_config(app_label) for model in app.get_models(): self.real_models.append(ModelState.from_model(model, exclude_rels=True)) # Populate the app registry with a stub for each application. app_labels = {model_state.app_label for model_state in models.values()} app_configs = [AppConfigStub(label) for label in sorted(real_apps + list(app_labels))] super(StateApps, self).__init__(app_configs) self.render_multiple(list(models.values()) + self.real_models) # If there are some lookups left, see if we can first resolve them # ourselves - sometimes fields are added after class_prepared is sent for lookup_model, operations in self._pending_lookups.items(): try: model = self.get_model(lookup_model[0], lookup_model[1]) except LookupError: app_label = "%s.%s" % (lookup_model[0], lookup_model[1]) if app_label == settings.AUTH_USER_MODEL and ignore_swappable: continue # Raise an error with a best-effort helpful message # (only for the first issue). Error message should look like: # "ValueError: Lookup failed for model referenced by # field migrations.Book.author: migrations.Author" msg = "Lookup failed for model referenced by field {field}: {model[0]}.{model[1]}" raise ValueError(msg.format(field=operations[0][1], model=lookup_model)) else: do_pending_lookups(model) def render_multiple(self, model_states): # We keep trying to render the models in a loop, ignoring invalid # base errors, until the size of the unrendered models doesn't # decrease by at least one, meaning there's a base dependency loop/ # missing base. if not model_states: return # Prevent that all model caches are expired for each render. self.ready = False unrendered_models = model_states while unrendered_models: new_unrendered_models = [] for model in unrendered_models: try: model.render(self) except InvalidBasesError: new_unrendered_models.append(model) if len(new_unrendered_models) == len(unrendered_models): self.ready = True raise InvalidBasesError( "Cannot resolve bases for %r\nThis can happen if you are inheriting models from an " "app with migrations (e.g. contrib.auth)\n in an app with no migrations; see " "https://docs.djangoproject.com/en/%s/topics/migrations/#dependencies " "for more" % (new_unrendered_models, get_docs_version()) ) unrendered_models = new_unrendered_models self.ready = True self.clear_cache() def clone(self): """ Return a clone of this registry, mainly used by the migration framework. """ clone = StateApps([], {}) clone.all_models = copy.deepcopy(self.all_models) clone.app_configs = copy.deepcopy(self.app_configs) # No need to actually clone them, they'll never change clone.real_models = self.real_models return clone def register_model(self, app_label, model): self.all_models[app_label][model._meta.model_name] = model if app_label not in self.app_configs: self.app_configs[app_label] = AppConfigStub(app_label) self.app_configs[app_label].models = OrderedDict() self.app_configs[app_label].models[model._meta.model_name] = model self.clear_cache() def unregister_model(self, app_label, model_name): try: del self.all_models[app_label][model_name] del self.app_configs[app_label].models[model_name] except KeyError: pass self.clear_cache() class ModelState(object): """ Represents a Django Model. We don't use the actual Model class as it's not designed to have its options changed - instead, we mutate this one and then render it into a Model as required. Note that while you are allowed to mutate .fields, you are not allowed to mutate the Field instances inside there themselves - you must instead assign new ones, as these are not detached during a clone. """ def __init__(self, app_label, name, fields, options=None, bases=None, managers=None): self.app_label = app_label self.name = force_text(name) self.fields = fields self.options = options or {} self.bases = bases or (models.Model, ) self.managers = managers or [] # Sanity-check that fields is NOT a dict. It must be ordered. if isinstance(self.fields, dict): raise ValueError("ModelState.fields cannot be a dict - it must be a list of 2-tuples.") # Sanity-check that fields are NOT already bound to a model. for name, field in fields: if hasattr(field, 'model'): raise ValueError( 'ModelState.fields cannot be bound to a model - "%s" is.' % name ) @cached_property def name_lower(self): return self.name.lower() @classmethod def from_model(cls, model, exclude_rels=False): """ Feed me a model, get a ModelState representing it out. """ # Deconstruct the fields fields = [] for field in model._meta.local_fields: if getattr(field, "rel", None) and exclude_rels: continue if isinstance(field, OrderWrt): continue name, path, args, kwargs = field.deconstruct() field_class = import_string(path) try: fields.append((name, field_class(*args, **kwargs))) except TypeError as e: raise TypeError("Couldn't reconstruct field %s on %s.%s: %s" % ( name, model._meta.app_label, model._meta.object_name, e, )) if not exclude_rels: for field in model._meta.local_many_to_many: name, path, args, kwargs = field.deconstruct() field_class = import_string(path) try: fields.append((name, field_class(*args, **kwargs))) except TypeError as e: raise TypeError("Couldn't reconstruct m2m field %s on %s: %s" % ( name, model._meta.object_name, e, )) # Extract the options options = {} for name in DEFAULT_NAMES: # Ignore some special options if name in ["apps", "app_label"]: continue elif name in model._meta.original_attrs: if name == "unique_together": ut = model._meta.original_attrs["unique_together"] options[name] = set(normalize_together(ut)) elif name == "index_together": it = model._meta.original_attrs["index_together"] options[name] = set(normalize_together(it)) else: options[name] = model._meta.original_attrs[name] # Force-convert all options to text_type (#23226) options = cls.force_text_recursive(options) # If we're ignoring relationships, remove all field-listing model # options (that option basically just means "make a stub model") if exclude_rels: for key in ["unique_together", "index_together", "order_with_respect_to"]: if key in options: del options[key] def flatten_bases(model): bases = [] for base in model.__bases__: if hasattr(base, "_meta") and base._meta.abstract: bases.extend(flatten_bases(base)) else: bases.append(base) return bases # We can't rely on __mro__ directly because we only want to flatten # abstract models and not the whole tree. However by recursing on # __bases__ we may end up with duplicates and ordering issues, we # therefore discard any duplicates and reorder the bases according # to their index in the MRO. flattened_bases = sorted(set(flatten_bases(model)), key=lambda x: model.__mro__.index(x)) # Make our record bases = tuple( ( "%s.%s" % (base._meta.app_label, base._meta.model_name) if hasattr(base, "_meta") else base ) for base in flattened_bases ) # Ensure at least one base inherits from models.Model if not any((isinstance(base, six.string_types) or issubclass(base, models.Model)) for base in bases): bases = (models.Model,) # Constructs all managers on the model managers = {} def reconstruct_manager(mgr): as_manager, manager_path, qs_path, args, kwargs = mgr.deconstruct() if as_manager: qs_class = import_string(qs_path) instance = qs_class.as_manager() else: manager_class = import_string(manager_path) instance = manager_class(*args, **kwargs) # We rely on the ordering of the creation_counter of the original # instance managers[mgr.name] = (mgr.creation_counter, instance) if hasattr(model, "_default_manager"): default_manager_name = model._default_manager.name # Make sure the default manager is always the first if model._default_manager.use_in_migrations: reconstruct_manager(model._default_manager) else: # Force this manager to be the first and thus default managers[default_manager_name] = (0, models.Manager()) # Sort all managers by their creation counter for _, manager, _ in sorted(model._meta.managers): if manager.name == "_base_manager" or not manager.use_in_migrations: continue reconstruct_manager(manager) # Sort all managers by their creation counter but take only name and # instance for further processing managers = [ (name, instance) for name, (cc, instance) in sorted(managers.items(), key=lambda v: v[1]) ] if managers == [(default_manager_name, models.Manager())]: managers = [] else: managers = [] # Construct the new ModelState return cls( model._meta.app_label, model._meta.object_name, fields, options, bases, managers, ) @classmethod def force_text_recursive(cls, value): if isinstance(value, six.string_types): return smart_text(value) elif isinstance(value, list): return [cls.force_text_recursive(x) for x in value] elif isinstance(value, tuple): return tuple(cls.force_text_recursive(x) for x in value) elif isinstance(value, set): return set(cls.force_text_recursive(x) for x in value) elif isinstance(value, dict): return { cls.force_text_recursive(k): cls.force_text_recursive(v) for k, v in value.items() } return value def construct_fields(self): "Deep-clone the fields using deconstruction" for name, field in self.fields: _, path, args, kwargs = field.deconstruct() field_class = import_string(path) yield name, field_class(*args, **kwargs) def construct_managers(self): "Deep-clone the managers using deconstruction" # Sort all managers by their creation counter sorted_managers = sorted(self.managers, key=lambda v: v[1].creation_counter) for mgr_name, manager in sorted_managers: as_manager, manager_path, qs_path, args, kwargs = manager.deconstruct() if as_manager: qs_class = import_string(qs_path) yield mgr_name, qs_class.as_manager() else: manager_class = import_string(manager_path) yield mgr_name, manager_class(*args, **kwargs) def clone(self): "Returns an exact copy of this ModelState" return self.__class__( app_label=self.app_label, name=self.name, fields=list(self.construct_fields()), options=dict(self.options), bases=self.bases, managers=list(self.construct_managers()), ) def render(self, apps): "Creates a Model object from our current state into the given apps" # First, make a Meta object meta_contents = {'app_label': self.app_label, "apps": apps} meta_contents.update(self.options) meta = type(str("Meta"), tuple(), meta_contents) # Then, work out our bases try: bases = tuple( (apps.get_model(base) if isinstance(base, six.string_types) else base) for base in self.bases ) except LookupError: raise InvalidBasesError("Cannot resolve one or more bases from %r" % (self.bases,)) # Turn fields into a dict for the body, add other bits body = dict(self.construct_fields()) body['Meta'] = meta body['__module__'] = "__fake__" # Restore managers body.update(self.construct_managers()) # Then, make a Model object (apps.register_model is called in __new__) return type( str(self.name), bases, body, ) def get_field_by_name(self, name): for fname, field in self.fields: if fname == name: return field raise ValueError("No field called %s on model %s" % (name, self.name)) def __repr__(self): return "" % (self.app_label, self.name) def __eq__(self, other): return ( (self.app_label == other.app_label) and (self.name == other.name) and (len(self.fields) == len(other.fields)) and all((k1 == k2 and (f1.deconstruct()[1:] == f2.deconstruct()[1:])) for (k1, f1), (k2, f2) in zip(self.fields, other.fields)) and (self.options == other.options) and (self.bases == other.bases) and (self.managers == other.managers) ) def __ne__(self, other): return not (self == other)