From d10c7bfe56f025ccc690721c9f13e7029b777b9c Mon Sep 17 00:00:00 2001 From: Ade Lee Date: Tue, 10 Aug 2021 18:13:54 -0400 Subject: [PATCH] Fixed #28401 -- Allowed hashlib.md5() calls to work with FIPS kernels. md5 is not an approved algorithm in FIPS mode, and trying to instantiate a hashlib.md5() will fail when the system is running in FIPS mode. md5 is allowed when in a non-security context. There is a plan to add a keyword parameter (usedforsecurity) to hashlib.md5() to annotate whether or not the instance is being used in a security context. In the case where it is not, the instantiation of md5 will be allowed. See https://bugs.python.org/issue9216 for more details. Some downstream python versions already support this parameter. To support these versions, a new encapsulation of md5() has been added. This encapsulation will pass through the usedforsecurity parameter in the case where the parameter is supported, and strip it if it is not. Co-authored-by: Mariusz Felisiak --- AUTHORS | 1 + django/contrib/auth/hashers.py | 6 +++--- django/contrib/staticfiles/storage.py | 8 ++++---- django/core/cache/backends/filebased.py | 8 +++++--- django/core/cache/utils.py | 4 ++-- django/db/backends/sqlite3/base.py | 3 ++- django/db/backends/utils.py | 4 ++-- django/test/runner.py | 4 ++-- django/utils/cache.py | 12 +++++++----- django/utils/crypto.py | 16 ++++++++++++++++ 10 files changed, 44 insertions(+), 22 deletions(-) diff --git a/AUTHORS b/AUTHORS index 967fe06449..55e6579840 100644 --- a/AUTHORS +++ b/AUTHORS @@ -19,6 +19,7 @@ answer newbie questions, and generally made Django that much better: Adam Johnson Adam Malinowski Adam Vandenberg + Ade Lee Adiyat Mubarak Adnan Umer Adrian Holovaty diff --git a/django/contrib/auth/hashers.py b/django/contrib/auth/hashers.py index be42cae3c0..7c865c26ef 100644 --- a/django/contrib/auth/hashers.py +++ b/django/contrib/auth/hashers.py @@ -11,7 +11,7 @@ from django.core.exceptions import ImproperlyConfigured from django.core.signals import setting_changed from django.dispatch import receiver from django.utils.crypto import ( - RANDOM_STRING_CHARS, constant_time_compare, get_random_string, pbkdf2, + RANDOM_STRING_CHARS, constant_time_compare, get_random_string, md5, pbkdf2, ) from django.utils.module_loading import import_string from django.utils.translation import gettext_noop as _ @@ -641,7 +641,7 @@ class MD5PasswordHasher(BasePasswordHasher): def encode(self, password, salt): self._check_encode_args(password, salt) - hash = hashlib.md5((salt + password).encode()).hexdigest() + hash = md5((salt + password).encode()).hexdigest() return "%s$%s$%s" % (self.algorithm, salt, hash) def decode(self, encoded): @@ -736,7 +736,7 @@ class UnsaltedMD5PasswordHasher(BasePasswordHasher): def encode(self, password, salt): if salt != '': raise ValueError('salt must be empty.') - return hashlib.md5(password.encode()).hexdigest() + return md5(password.encode()).hexdigest() def decode(self, encoded): return { diff --git a/django/contrib/staticfiles/storage.py b/django/contrib/staticfiles/storage.py index 7cf43ed6ab..5fa088a29a 100644 --- a/django/contrib/staticfiles/storage.py +++ b/django/contrib/staticfiles/storage.py @@ -1,4 +1,3 @@ -import hashlib import json import os import posixpath @@ -10,6 +9,7 @@ from django.contrib.staticfiles.utils import check_settings, matches_patterns from django.core.exceptions import ImproperlyConfigured from django.core.files.base import ContentFile from django.core.files.storage import FileSystemStorage, get_storage_class +from django.utils.crypto import md5 from django.utils.functional import LazyObject @@ -89,10 +89,10 @@ class HashedFilesMixin: """ if content is None: return None - md5 = hashlib.md5() + hasher = md5(usedforsecurity=False) for chunk in content.chunks(): - md5.update(chunk) - return md5.hexdigest()[:12] + hasher.update(chunk) + return hasher.hexdigest()[:12] def hashed_name(self, name, content=None, filename=None): # `filename` is the name of file to hash if `content` isn't given. diff --git a/django/core/cache/backends/filebased.py b/django/core/cache/backends/filebased.py index dcba36610e..fc99d11687 100644 --- a/django/core/cache/backends/filebased.py +++ b/django/core/cache/backends/filebased.py @@ -1,6 +1,5 @@ "File-based cache backend" import glob -import hashlib import os import pickle import random @@ -11,6 +10,7 @@ import zlib from django.core.cache.backends.base import DEFAULT_TIMEOUT, BaseCache from django.core.files import locks from django.core.files.move import file_move_safe +from django.utils.crypto import md5 class FileBasedCache(BaseCache): @@ -128,8 +128,10 @@ class FileBasedCache(BaseCache): root cache path joined with the md5sum of the key and a suffix. """ key = self.make_and_validate_key(key, version=version) - return os.path.join(self._dir, ''.join( - [hashlib.md5(key.encode()).hexdigest(), self.cache_suffix])) + return os.path.join(self._dir, ''.join([ + md5(key.encode(), usedforsecurity=False).hexdigest(), + self.cache_suffix, + ])) def clear(self): """ diff --git a/django/core/cache/utils.py b/django/core/cache/utils.py index 2aead84d60..d41960f6e4 100644 --- a/django/core/cache/utils.py +++ b/django/core/cache/utils.py @@ -1,10 +1,10 @@ -import hashlib +from django.utils.crypto import md5 TEMPLATE_FRAGMENT_KEY_TEMPLATE = 'template.cache.%s.%s' def make_template_fragment_key(fragment_name, vary_on=None): - hasher = hashlib.md5() + hasher = md5(usedforsecurity=False) if vary_on is not None: for arg in vary_on: hasher.update(str(arg).encode()) diff --git a/django/db/backends/sqlite3/base.py b/django/db/backends/sqlite3/base.py index ddf5f40c8e..666b367be7 100644 --- a/django/db/backends/sqlite3/base.py +++ b/django/db/backends/sqlite3/base.py @@ -22,6 +22,7 @@ from django.db.backends.base.base import ( ) from django.utils import timezone from django.utils.asyncio import async_unsafe +from django.utils.crypto import md5 from django.utils.dateparse import parse_datetime, parse_time from django.utils.duration import duration_microseconds from django.utils.regex_helper import _lazy_re_compile @@ -233,7 +234,7 @@ class DatabaseWrapper(BaseDatabaseWrapper): create_deterministic_function('LN', 1, none_guard(math.log)) create_deterministic_function('LOG', 2, none_guard(lambda x, y: math.log(y, x))) create_deterministic_function('LPAD', 3, _sqlite_lpad) - create_deterministic_function('MD5', 1, none_guard(lambda x: hashlib.md5(x.encode()).hexdigest())) + create_deterministic_function('MD5', 1, none_guard(lambda x: md5(x.encode()).hexdigest())) create_deterministic_function('MOD', 2, none_guard(math.fmod)) create_deterministic_function('PI', 0, lambda: math.pi) create_deterministic_function('POWER', 2, none_guard(operator.pow)) diff --git a/django/db/backends/utils.py b/django/db/backends/utils.py index c342cf79b5..eda7159a41 100644 --- a/django/db/backends/utils.py +++ b/django/db/backends/utils.py @@ -1,12 +1,12 @@ import datetime import decimal import functools -import hashlib import logging import time from contextlib import contextmanager from django.db import NotSupportedError +from django.utils.crypto import md5 logger = logging.getLogger('django.db.backends') @@ -216,7 +216,7 @@ def names_digest(*args, length): Generate a 32-bit digest of a set of arguments that can be used to shorten identifying names. """ - h = hashlib.md5() + h = md5(usedforsecurity=False) for arg in args: h.update(arg.encode()) return h.hexdigest()[:length] diff --git a/django/test/runner.py b/django/test/runner.py index 34480cf103..09ac4e142a 100644 --- a/django/test/runner.py +++ b/django/test/runner.py @@ -1,7 +1,6 @@ import argparse import ctypes import faulthandler -import hashlib import io import itertools import logging @@ -26,6 +25,7 @@ from django.test.utils import ( setup_databases as _setup_databases, setup_test_environment, teardown_databases as _teardown_databases, teardown_test_environment, ) +from django.utils.crypto import new_hash from django.utils.datastructures import OrderedSet from django.utils.deprecation import RemovedInDjango50Warning @@ -509,7 +509,7 @@ class Shuffler: @classmethod def _hash_text(cls, text): - h = hashlib.new(cls.hash_algorithm) + h = new_hash(cls.hash_algorithm, usedforsecurity=False) h.update(text.encode('utf-8')) return h.hexdigest() diff --git a/django/utils/cache.py b/django/utils/cache.py index bb756fe60c..c0e47e0e42 100644 --- a/django/utils/cache.py +++ b/django/utils/cache.py @@ -16,13 +16,13 @@ cache keys to prevent delivery of wrong content. An example: i18n middleware would need to distinguish caches by the "Accept-language" header. """ -import hashlib import time from collections import defaultdict from django.conf import settings from django.core.cache import caches from django.http import HttpResponse, HttpResponseNotModified +from django.utils.crypto import md5 from django.utils.http import ( http_date, parse_etags, parse_http_date_safe, quote_etag, ) @@ -118,7 +118,9 @@ def get_max_age(response): def set_response_etag(response): if not response.streaming and response.content: - response.headers['ETag'] = quote_etag(hashlib.md5(response.content).hexdigest()) + response.headers['ETag'] = quote_etag( + md5(response.content, usedforsecurity=False).hexdigest(), + ) return response @@ -325,12 +327,12 @@ def _i18n_cache_key_suffix(request, cache_key): def _generate_cache_key(request, method, headerlist, key_prefix): """Return a cache key from the headers given in the header list.""" - ctx = hashlib.md5() + ctx = md5(usedforsecurity=False) for header in headerlist: value = request.META.get(header) if value is not None: ctx.update(value.encode()) - url = hashlib.md5(request.build_absolute_uri().encode('ascii')) + url = md5(request.build_absolute_uri().encode('ascii'), usedforsecurity=False) cache_key = 'views.decorators.cache.cache_page.%s.%s.%s.%s' % ( key_prefix, method, url.hexdigest(), ctx.hexdigest()) return _i18n_cache_key_suffix(request, cache_key) @@ -338,7 +340,7 @@ def _generate_cache_key(request, method, headerlist, key_prefix): def _generate_cache_header_key(key_prefix, request): """Return a cache key for the header cache.""" - url = hashlib.md5(request.build_absolute_uri().encode('ascii')) + url = md5(request.build_absolute_uri().encode('ascii'), usedforsecurity=False) cache_key = 'views.decorators.cache.cache_header.%s.%s' % ( key_prefix, url.hexdigest()) return _i18n_cache_key_suffix(request, cache_key) diff --git a/django/utils/crypto.py b/django/utils/crypto.py index 9d76f950b2..2af58fda6e 100644 --- a/django/utils/crypto.py +++ b/django/utils/crypto.py @@ -7,6 +7,7 @@ import secrets from django.conf import settings from django.utils.encoding import force_bytes +from django.utils.inspect import func_supports_parameter class InvalidAlgorithm(ValueError): @@ -74,3 +75,18 @@ def pbkdf2(password, salt, iterations, dklen=0, digest=None): password = force_bytes(password) salt = force_bytes(salt) return hashlib.pbkdf2_hmac(digest().name, password, salt, iterations, dklen) + + +# TODO: Remove when dropping support for PY38. inspect.signature() is used to +# detect whether the usedforsecurity argument is available as this fix may also +# have been applied by downstream package maintainers to other versions in +# their repositories. +if func_supports_parameter(hashlib.md5, 'usedforsecurity'): + md5 = hashlib.md5 + new_hash = hashlib.new +else: + def md5(data=b'', *, usedforsecurity=True): + return hashlib.md5(data) + + def new_hash(hash_algorithm, *, usedforsecurity=True): + return hashlib.new(hash_algorithm)