schema.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434
  1. import copy
  2. from decimal import Decimal
  3. from django.apps.registry import Apps
  4. from django.db import NotSupportedError
  5. from django.db.backends.base.schema import BaseDatabaseSchemaEditor
  6. from django.db.backends.ddl_references import Statement
  7. from django.db.backends.utils import strip_quotes
  8. from django.db.models import UniqueConstraint
  9. from django.db.transaction import atomic
  10. class DatabaseSchemaEditor(BaseDatabaseSchemaEditor):
  11. sql_delete_table = "DROP TABLE %(table)s"
  12. sql_create_fk = None
  13. sql_create_inline_fk = "REFERENCES %(to_table)s (%(to_column)s) DEFERRABLE INITIALLY DEFERRED"
  14. sql_create_unique = "CREATE UNIQUE INDEX %(name)s ON %(table)s (%(columns)s)"
  15. sql_delete_unique = "DROP INDEX %(name)s"
  16. def __enter__(self):
  17. # Some SQLite schema alterations need foreign key constraints to be
  18. # disabled. Enforce it here for the duration of the schema edition.
  19. if not self.connection.disable_constraint_checking():
  20. raise NotSupportedError(
  21. 'SQLite schema editor cannot be used while foreign key '
  22. 'constraint checks are enabled. Make sure to disable them '
  23. 'before entering a transaction.atomic() context because '
  24. 'SQLite does not support disabling them in the middle of '
  25. 'a multi-statement transaction.'
  26. )
  27. return super().__enter__()
  28. def __exit__(self, exc_type, exc_value, traceback):
  29. self.connection.check_constraints()
  30. super().__exit__(exc_type, exc_value, traceback)
  31. self.connection.enable_constraint_checking()
  32. def quote_value(self, value):
  33. # The backend "mostly works" without this function and there are use
  34. # cases for compiling Python without the sqlite3 libraries (e.g.
  35. # security hardening).
  36. try:
  37. import sqlite3
  38. value = sqlite3.adapt(value)
  39. except ImportError:
  40. pass
  41. except sqlite3.ProgrammingError:
  42. pass
  43. # Manual emulation of SQLite parameter quoting
  44. if isinstance(value, bool):
  45. return str(int(value))
  46. elif isinstance(value, (Decimal, float, int)):
  47. return str(value)
  48. elif isinstance(value, str):
  49. return "'%s'" % value.replace("\'", "\'\'")
  50. elif value is None:
  51. return "NULL"
  52. elif isinstance(value, (bytes, bytearray, memoryview)):
  53. # Bytes are only allowed for BLOB fields, encoded as string
  54. # literals containing hexadecimal data and preceded by a single "X"
  55. # character.
  56. return "X'%s'" % value.hex()
  57. else:
  58. raise ValueError("Cannot quote parameter value %r of type %s" % (value, type(value)))
  59. def _is_referenced_by_fk_constraint(self, table_name, column_name=None, ignore_self=False):
  60. """
  61. Return whether or not the provided table name is referenced by another
  62. one. If `column_name` is specified, only references pointing to that
  63. column are considered. If `ignore_self` is True, self-referential
  64. constraints are ignored.
  65. """
  66. with self.connection.cursor() as cursor:
  67. for other_table in self.connection.introspection.get_table_list(cursor):
  68. if ignore_self and other_table.name == table_name:
  69. continue
  70. constraints = self.connection.introspection._get_foreign_key_constraints(cursor, other_table.name)
  71. for constraint in constraints.values():
  72. constraint_table, constraint_column = constraint['foreign_key']
  73. if (constraint_table == table_name and
  74. (column_name is None or constraint_column == column_name)):
  75. return True
  76. return False
  77. def alter_db_table(self, model, old_db_table, new_db_table, disable_constraints=True):
  78. if (not self.connection.features.supports_atomic_references_rename and
  79. disable_constraints and self._is_referenced_by_fk_constraint(old_db_table)):
  80. if self.connection.in_atomic_block:
  81. raise NotSupportedError((
  82. 'Renaming the %r table while in a transaction is not '
  83. 'supported on SQLite < 3.26 because it would break referential '
  84. 'integrity. Try adding `atomic = False` to the Migration class.'
  85. ) % old_db_table)
  86. self.connection.enable_constraint_checking()
  87. super().alter_db_table(model, old_db_table, new_db_table)
  88. self.connection.disable_constraint_checking()
  89. else:
  90. super().alter_db_table(model, old_db_table, new_db_table)
  91. def alter_field(self, model, old_field, new_field, strict=False):
  92. if not self._field_should_be_altered(old_field, new_field):
  93. return
  94. old_field_name = old_field.name
  95. table_name = model._meta.db_table
  96. _, old_column_name = old_field.get_attname_column()
  97. if (new_field.name != old_field_name and
  98. not self.connection.features.supports_atomic_references_rename and
  99. self._is_referenced_by_fk_constraint(table_name, old_column_name, ignore_self=True)):
  100. if self.connection.in_atomic_block:
  101. raise NotSupportedError((
  102. 'Renaming the %r.%r column while in a transaction is not '
  103. 'supported on SQLite < 3.26 because it would break referential '
  104. 'integrity. Try adding `atomic = False` to the Migration class.'
  105. ) % (model._meta.db_table, old_field_name))
  106. with atomic(self.connection.alias):
  107. super().alter_field(model, old_field, new_field, strict=strict)
  108. # Follow SQLite's documented procedure for performing changes
  109. # that don't affect the on-disk content.
  110. # https://sqlite.org/lang_altertable.html#otheralter
  111. with self.connection.cursor() as cursor:
  112. schema_version = cursor.execute('PRAGMA schema_version').fetchone()[0]
  113. cursor.execute('PRAGMA writable_schema = 1')
  114. references_template = ' REFERENCES "%s" ("%%s") ' % table_name
  115. new_column_name = new_field.get_attname_column()[1]
  116. search = references_template % old_column_name
  117. replacement = references_template % new_column_name
  118. cursor.execute('UPDATE sqlite_master SET sql = replace(sql, %s, %s)', (search, replacement))
  119. cursor.execute('PRAGMA schema_version = %d' % (schema_version + 1))
  120. cursor.execute('PRAGMA writable_schema = 0')
  121. # The integrity check will raise an exception and rollback
  122. # the transaction if the sqlite_master updates corrupt the
  123. # database.
  124. cursor.execute('PRAGMA integrity_check')
  125. # Perform a VACUUM to refresh the database representation from
  126. # the sqlite_master table.
  127. with self.connection.cursor() as cursor:
  128. cursor.execute('VACUUM')
  129. else:
  130. super().alter_field(model, old_field, new_field, strict=strict)
  131. def _remake_table(self, model, create_field=None, delete_field=None, alter_field=None):
  132. """
  133. Shortcut to transform a model from old_model into new_model
  134. This follows the correct procedure to perform non-rename or column
  135. addition operations based on SQLite's documentation
  136. https://www.sqlite.org/lang_altertable.html#caution
  137. The essential steps are:
  138. 1. Create a table with the updated definition called "new__app_model"
  139. 2. Copy the data from the existing "app_model" table to the new table
  140. 3. Drop the "app_model" table
  141. 4. Rename the "new__app_model" table to "app_model"
  142. 5. Restore any index of the previous "app_model" table.
  143. """
  144. # Self-referential fields must be recreated rather than copied from
  145. # the old model to ensure their remote_field.field_name doesn't refer
  146. # to an altered field.
  147. def is_self_referential(f):
  148. return f.is_relation and f.remote_field.model is model
  149. # Work out the new fields dict / mapping
  150. body = {
  151. f.name: f.clone() if is_self_referential(f) else f
  152. for f in model._meta.local_concrete_fields
  153. }
  154. # Since mapping might mix column names and default values,
  155. # its values must be already quoted.
  156. mapping = {f.column: self.quote_name(f.column) for f in model._meta.local_concrete_fields}
  157. # This maps field names (not columns) for things like unique_together
  158. rename_mapping = {}
  159. # If any of the new or altered fields is introducing a new PK,
  160. # remove the old one
  161. restore_pk_field = None
  162. if getattr(create_field, 'primary_key', False) or (
  163. alter_field and getattr(alter_field[1], 'primary_key', False)):
  164. for name, field in list(body.items()):
  165. if field.primary_key:
  166. field.primary_key = False
  167. restore_pk_field = field
  168. if field.auto_created:
  169. del body[name]
  170. del mapping[field.column]
  171. # Add in any created fields
  172. if create_field:
  173. body[create_field.name] = create_field
  174. # Choose a default and insert it into the copy map
  175. if not create_field.many_to_many and create_field.concrete:
  176. mapping[create_field.column] = self.quote_value(
  177. self.effective_default(create_field)
  178. )
  179. # Add in any altered fields
  180. if alter_field:
  181. old_field, new_field = alter_field
  182. body.pop(old_field.name, None)
  183. mapping.pop(old_field.column, None)
  184. body[new_field.name] = new_field
  185. if old_field.null and not new_field.null:
  186. case_sql = "coalesce(%(col)s, %(default)s)" % {
  187. 'col': self.quote_name(old_field.column),
  188. 'default': self.quote_value(self.effective_default(new_field))
  189. }
  190. mapping[new_field.column] = case_sql
  191. else:
  192. mapping[new_field.column] = self.quote_name(old_field.column)
  193. rename_mapping[old_field.name] = new_field.name
  194. # Remove any deleted fields
  195. if delete_field:
  196. del body[delete_field.name]
  197. del mapping[delete_field.column]
  198. # Remove any implicit M2M tables
  199. if delete_field.many_to_many and delete_field.remote_field.through._meta.auto_created:
  200. return self.delete_model(delete_field.remote_field.through)
  201. # Work inside a new app registry
  202. apps = Apps()
  203. # Work out the new value of unique_together, taking renames into
  204. # account
  205. unique_together = [
  206. [rename_mapping.get(n, n) for n in unique]
  207. for unique in model._meta.unique_together
  208. ]
  209. # Work out the new value for index_together, taking renames into
  210. # account
  211. index_together = [
  212. [rename_mapping.get(n, n) for n in index]
  213. for index in model._meta.index_together
  214. ]
  215. indexes = model._meta.indexes
  216. if delete_field:
  217. indexes = [
  218. index for index in indexes
  219. if delete_field.name not in index.fields
  220. ]
  221. constraints = list(model._meta.constraints)
  222. # Provide isolated instances of the fields to the new model body so
  223. # that the existing model's internals aren't interfered with when
  224. # the dummy model is constructed.
  225. body_copy = copy.deepcopy(body)
  226. # Construct a new model with the new fields to allow self referential
  227. # primary key to resolve to. This model won't ever be materialized as a
  228. # table and solely exists for foreign key reference resolution purposes.
  229. # This wouldn't be required if the schema editor was operating on model
  230. # states instead of rendered models.
  231. meta_contents = {
  232. 'app_label': model._meta.app_label,
  233. 'db_table': model._meta.db_table,
  234. 'unique_together': unique_together,
  235. 'index_together': index_together,
  236. 'indexes': indexes,
  237. 'constraints': constraints,
  238. 'apps': apps,
  239. }
  240. meta = type("Meta", (), meta_contents)
  241. body_copy['Meta'] = meta
  242. body_copy['__module__'] = model.__module__
  243. type(model._meta.object_name, model.__bases__, body_copy)
  244. # Construct a model with a renamed table name.
  245. body_copy = copy.deepcopy(body)
  246. meta_contents = {
  247. 'app_label': model._meta.app_label,
  248. 'db_table': 'new__%s' % strip_quotes(model._meta.db_table),
  249. 'unique_together': unique_together,
  250. 'index_together': index_together,
  251. 'indexes': indexes,
  252. 'constraints': constraints,
  253. 'apps': apps,
  254. }
  255. meta = type("Meta", (), meta_contents)
  256. body_copy['Meta'] = meta
  257. body_copy['__module__'] = model.__module__
  258. new_model = type('New%s' % model._meta.object_name, model.__bases__, body_copy)
  259. # Create a new table with the updated schema.
  260. self.create_model(new_model)
  261. # Copy data from the old table into the new table
  262. self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
  263. self.quote_name(new_model._meta.db_table),
  264. ', '.join(self.quote_name(x) for x in mapping),
  265. ', '.join(mapping.values()),
  266. self.quote_name(model._meta.db_table),
  267. ))
  268. # Delete the old table to make way for the new
  269. self.delete_model(model, handle_autom2m=False)
  270. # Rename the new table to take way for the old
  271. self.alter_db_table(
  272. new_model, new_model._meta.db_table, model._meta.db_table,
  273. disable_constraints=False,
  274. )
  275. # Run deferred SQL on correct table
  276. for sql in self.deferred_sql:
  277. self.execute(sql)
  278. self.deferred_sql = []
  279. # Fix any PK-removed field
  280. if restore_pk_field:
  281. restore_pk_field.primary_key = True
  282. def delete_model(self, model, handle_autom2m=True):
  283. if handle_autom2m:
  284. super().delete_model(model)
  285. else:
  286. # Delete the table (and only that)
  287. self.execute(self.sql_delete_table % {
  288. "table": self.quote_name(model._meta.db_table),
  289. })
  290. # Remove all deferred statements referencing the deleted table.
  291. for sql in list(self.deferred_sql):
  292. if isinstance(sql, Statement) and sql.references_table(model._meta.db_table):
  293. self.deferred_sql.remove(sql)
  294. def add_field(self, model, field):
  295. """
  296. Create a field on a model. Usually involves adding a column, but may
  297. involve adding a table instead (for M2M fields).
  298. """
  299. # Special-case implicit M2M tables
  300. if field.many_to_many and field.remote_field.through._meta.auto_created:
  301. return self.create_model(field.remote_field.through)
  302. self._remake_table(model, create_field=field)
  303. def remove_field(self, model, field):
  304. """
  305. Remove a field from a model. Usually involves deleting a column,
  306. but for M2Ms may involve deleting a table.
  307. """
  308. # M2M fields are a special case
  309. if field.many_to_many:
  310. # For implicit M2M tables, delete the auto-created table
  311. if field.remote_field.through._meta.auto_created:
  312. self.delete_model(field.remote_field.through)
  313. # For explicit "through" M2M fields, do nothing
  314. # For everything else, remake.
  315. else:
  316. # It might not actually have a column behind it
  317. if field.db_parameters(connection=self.connection)['type'] is None:
  318. return
  319. self._remake_table(model, delete_field=field)
  320. def _alter_field(self, model, old_field, new_field, old_type, new_type,
  321. old_db_params, new_db_params, strict=False):
  322. """Perform a "physical" (non-ManyToMany) field update."""
  323. # Use "ALTER TABLE ... RENAME COLUMN" if only the column name
  324. # changed and there aren't any constraints.
  325. if (self.connection.features.can_alter_table_rename_column and
  326. old_field.column != new_field.column and
  327. self.column_sql(model, old_field) == self.column_sql(model, new_field) and
  328. not (old_field.remote_field and old_field.db_constraint or
  329. new_field.remote_field and new_field.db_constraint)):
  330. return self.execute(self._rename_field_sql(model._meta.db_table, old_field, new_field, new_type))
  331. # Alter by remaking table
  332. self._remake_table(model, alter_field=(old_field, new_field))
  333. # Rebuild tables with FKs pointing to this field.
  334. if new_field.unique and old_type != new_type:
  335. related_models = set()
  336. opts = new_field.model._meta
  337. for remote_field in opts.related_objects:
  338. # Ignore self-relationship since the table was already rebuilt.
  339. if remote_field.related_model == model:
  340. continue
  341. if not remote_field.many_to_many:
  342. if remote_field.field_name == new_field.name:
  343. related_models.add(remote_field.related_model)
  344. elif new_field.primary_key and remote_field.through._meta.auto_created:
  345. related_models.add(remote_field.through)
  346. if new_field.primary_key:
  347. for many_to_many in opts.many_to_many:
  348. # Ignore self-relationship since the table was already rebuilt.
  349. if many_to_many.related_model == model:
  350. continue
  351. if many_to_many.remote_field.through._meta.auto_created:
  352. related_models.add(many_to_many.remote_field.through)
  353. for related_model in related_models:
  354. self._remake_table(related_model)
  355. def _alter_many_to_many(self, model, old_field, new_field, strict):
  356. """Alter M2Ms to repoint their to= endpoints."""
  357. if old_field.remote_field.through._meta.db_table == new_field.remote_field.through._meta.db_table:
  358. # The field name didn't change, but some options did; we have to propagate this altering.
  359. self._remake_table(
  360. old_field.remote_field.through,
  361. alter_field=(
  362. # We need the field that points to the target model, so we can tell alter_field to change it -
  363. # this is m2m_reverse_field_name() (as opposed to m2m_field_name, which points to our model)
  364. old_field.remote_field.through._meta.get_field(old_field.m2m_reverse_field_name()),
  365. new_field.remote_field.through._meta.get_field(new_field.m2m_reverse_field_name()),
  366. ),
  367. )
  368. return
  369. # Make a new through table
  370. self.create_model(new_field.remote_field.through)
  371. # Copy the data across
  372. self.execute("INSERT INTO %s (%s) SELECT %s FROM %s" % (
  373. self.quote_name(new_field.remote_field.through._meta.db_table),
  374. ', '.join([
  375. "id",
  376. new_field.m2m_column_name(),
  377. new_field.m2m_reverse_name(),
  378. ]),
  379. ', '.join([
  380. "id",
  381. old_field.m2m_column_name(),
  382. old_field.m2m_reverse_name(),
  383. ]),
  384. self.quote_name(old_field.remote_field.through._meta.db_table),
  385. ))
  386. # Delete the old through table
  387. self.delete_model(old_field.remote_field.through)
  388. def add_constraint(self, model, constraint):
  389. if isinstance(constraint, UniqueConstraint) and constraint.condition:
  390. super().add_constraint(model, constraint)
  391. else:
  392. self._remake_table(model)
  393. def remove_constraint(self, model, constraint):
  394. if isinstance(constraint, UniqueConstraint) and constraint.condition:
  395. super().remove_constraint(model, constraint)
  396. else:
  397. self._remake_table(model)
  398. def _collate_sql(self, collation):
  399. return ' COLLATE ' + collation