Fixed #16501 -- Added an allow_unicode parameter to SlugField.

Thanks Flavio Curella and Berker Peksag for the initial patch.

django/contrib/admin/options.py

@@ -571,6 +571,7 @@ class ModelAdmin(BaseModelAdmin):
             'actions%s.js' % extra,
             'urlify.js',
             'prepopulate%s.js' % extra,
+            'vendor/xregexp/xregexp.min.js',
         ]
         return forms.Media(js=[static('admin/js/%s' % url) for url in js])
 

django/contrib/admin/static/admin/js/actions.min.js

@@ -1,6 +1,6 @@
-(function(a){var f;a.fn.actions=function(q){var b=a.extend({},a.fn.actions.defaults,q),g=a(this),e=!1,m=function(c){c?k():l();a(g).prop("checked",c).parent().parent().toggleClass(b.selectedClass,c)},h=function(){var c=a(g).filter(":checked").length;a(b.counterContainer).html(interpolate(ngettext("%(sel)s of %(cnt)s selected","%(sel)s of %(cnt)s selected",c),{sel:c,cnt:_actions_icnt},!0));a(b.allToggle).prop("checked",function(){var a;c==g.length?(a=!0,k()):(a=!1,n());return a})},k=function(){a(b.acrossClears).hide();
+(function(a){var f;a.fn.actions=function(q){var b=a.extend({},a.fn.actions.defaults,q),g=a(this),e=!1,k=function(){a(b.acrossClears).hide();a(b.acrossQuestions).show();a(b.allContainer).hide()},l=function(){a(b.acrossClears).show();a(b.acrossQuestions).hide();a(b.actionContainer).toggleClass(b.selectedClass);a(b.allContainer).show();a(b.counterContainer).hide()},m=function(){a(b.acrossClears).hide();a(b.acrossQuestions).hide();a(b.allContainer).hide();a(b.counterContainer).show()},n=function(){m();
-a(b.acrossQuestions).show();a(b.allContainer).hide()},p=function(){a(b.acrossClears).show();a(b.acrossQuestions).hide();a(b.actionContainer).toggleClass(b.selectedClass);a(b.allContainer).show();a(b.counterContainer).hide()},l=function(){a(b.acrossClears).hide();a(b.acrossQuestions).hide();a(b.allContainer).hide();a(b.counterContainer).show()},n=function(){l();a(b.acrossInput).val(0);a(b.actionContainer).removeClass(b.selectedClass)};a(b.counterContainer).show();a(this).filter(":checked").each(function(c){a(this).parent().parent().toggleClass(b.selectedClass);
+a(b.acrossInput).val(0);a(b.actionContainer).removeClass(b.selectedClass)},p=function(c){c?k():m();a(g).prop("checked",c).parent().parent().toggleClass(b.selectedClass,c)},h=function(){var c=a(g).filter(":checked").length;a(b.counterContainer).html(interpolate(ngettext("%(sel)s of %(cnt)s selected","%(sel)s of %(cnt)s selected",c),{sel:c,cnt:_actions_icnt},!0));a(b.allToggle).prop("checked",function(){var a;c===g.length?(a=!0,k()):(a=!1,n());return a})};a(b.counterContainer).show();a(this).filter(":checked").each(function(c){a(this).parent().parent().toggleClass(b.selectedClass);
-h();1==a(b.acrossInput).val()&&p()});a(b.allToggle).show().click(function(){m(a(this).prop("checked"));h()});a("a",b.acrossQuestions).click(function(c){c.preventDefault();a(b.acrossInput).val(1);p()});a("a",b.acrossClears).click(function(c){c.preventDefault();a(b.allToggle).prop("checked",!1);n();m(0);h()});f=null;a(g).click(function(c){c||(c=window.event);var d=c.target?c.target:c.srcElement;if(f&&a.data(f)!=a.data(d)&&!0===c.shiftKey){var e=!1;a(f).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,
+h();1===a(b.acrossInput).val()&&l()});a(b.allToggle).show().click(function(){p(a(this).prop("checked"));h()});a("a",b.acrossQuestions).click(function(c){c.preventDefault();a(b.acrossInput).val(1);l()});a("a",b.acrossClears).click(function(c){c.preventDefault();a(b.allToggle).prop("checked",!1);n();p(0);h()});f=null;a(g).click(function(c){c||(c=window.event);var d=c.target?c.target:c.srcElement;if(f&&a.data(f)!==a.data(d)&&!0===c.shiftKey){var e=!1;a(f).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,
-d.checked);a(g).each(function(){if(a.data(this)==a.data(f)||a.data(this)==a.data(d))e=e?!1:!0;e&&a(this).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,d.checked)})}a(d).parent().parent().toggleClass(b.selectedClass,d.checked);f=d;h()});a("form#changelist-form table#result_list tr").find("td:gt(0) :input").change(function(){e=!0});a('form#changelist-form button[name="index"]').click(function(a){if(e)return confirm(gettext("You have unsaved changes on individual editable fields. If you run an action, your unsaved changes will be lost."))});
+d.checked);a(g).each(function(){if(a.data(this)===a.data(f)||a.data(this)===a.data(d))e=e?!1:!0;e&&a(this).prop("checked",d.checked).parent().parent().toggleClass(b.selectedClass,d.checked)})}a(d).parent().parent().toggleClass(b.selectedClass,d.checked);f=d;h()});a("form#changelist-form table#result_list tr").find("td:gt(0) :input").change(function(){e=!0});a('form#changelist-form button[name="index"]').click(function(a){if(e)return confirm(gettext("You have unsaved changes on individual editable fields. If you run an action, your unsaved changes will be lost."))});
 a('form#changelist-form input[name="_save"]').click(function(c){var d=!1;a("select option:selected",b.actionContainer).each(function(){a(this).val()&&(d=!0)});if(d)return e?confirm(gettext("You have selected an action, but you haven't saved your changes to individual fields yet. Please click OK to save. You'll need to re-run the action.")):confirm(gettext("You have selected an action, and you haven't made any changes on individual fields. You're probably looking for the Go button rather than the Save button."))})};
 a.fn.actions.defaults={actionContainer:"div.actions",counterContainer:"span.action-counter",allContainer:"div.actions span.all",acrossInput:"div.actions input.select-across",acrossQuestions:"div.actions span.question",acrossClears:"div.actions span.clear",allToggle:"#action-toggle",selectedClass:"selected"}})(django.jQuery);

django/contrib/admin/static/admin/js/inlines.min.js

@@ -1,9 +1,9 @@
-(function(c){c.fn.formset=function(b){var a=c.extend({},c.fn.formset.defaults,b),d=c(this);b=d.parent();var k=function(a,g,l){var h=new RegExp("("+g+"-(\\d+|__prefix__))");g=g+"-"+l;c(a).prop("for")&&c(a).prop("for",c(a).prop("for").replace(h,g));a.id&&(a.id=a.id.replace(h,g));a.name&&(a.name=a.name.replace(h,g))},f=c("#id_"+a.prefix+"-TOTAL_FORMS").prop("autocomplete","off"),l=parseInt(f.val(),10),g=c("#id_"+a.prefix+"-MAX_NUM_FORMS").prop("autocomplete","off"),f=""===g.val()||0<g.val()-f.val();
+(function(c){c.fn.formset=function(b){var a=c.extend({},c.fn.formset.defaults,b),d=c(this);b=d.parent();var k=function(a,g,l){var h=new RegExp("("+g+"-(\\d+|__prefix__))");g=g+"-"+l;c(a).prop("for")&&c(a).prop("for",c(a).prop("for").replace(h,g));a.id&&(a.id=a.id.replace(h,g));a.name&&(a.name=a.name.replace(h,g))},e=c("#id_"+a.prefix+"-TOTAL_FORMS").prop("autocomplete","off"),l=parseInt(e.val(),10),g=c("#id_"+a.prefix+"-MAX_NUM_FORMS").prop("autocomplete","off"),e=""===g.val()||0<g.val()-e.val();
-d.each(function(g){c(this).not("."+a.emptyCssClass).addClass(a.formCssClass)});if(d.length&&f){var h;"TR"==d.prop("tagName")?(d=this.eq(-1).children().length,b.append('<tr class="'+a.addCssClass+'"><td colspan="'+d+'"><a href="javascript:void(0)">'+a.addText+"</a></tr>"),h=b.find("tr:last a")):(d.filter(":last").after('<div class="'+a.addCssClass+'"><a href="javascript:void(0)">'+a.addText+"</a></div>"),h=d.filter(":last").next().find("a"));h.click(function(b){b.preventDefault();var d=c("#id_"+a.prefix+
+d.each(function(g){c(this).not("."+a.emptyCssClass).addClass(a.formCssClass)});if(d.length&&e){var h;"TR"===d.prop("tagName")?(d=this.eq(-1).children().length,b.append('<tr class="'+a.addCssClass+'"><td colspan="'+d+'"><a href="javascript:void(0)">'+a.addText+"</a></tr>"),h=b.find("tr:last a")):(d.filter(":last").after('<div class="'+a.addCssClass+'"><a href="javascript:void(0)">'+a.addText+"</a></div>"),h=d.filter(":last").next().find("a"));h.click(function(b){b.preventDefault();var d=c("#id_"+a.prefix+
-"-TOTAL_FORMS");b=c("#"+a.prefix+"-empty");var e=b.clone(!0);e.removeClass(a.emptyCssClass).addClass(a.formCssClass).attr("id",a.prefix+"-"+l);e.is("tr")?e.children(":last").append('<div><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+"</a></div>"):e.is("ul")||e.is("ol")?e.append('<li><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+"</a></li>"):e.children(":first").append('<span><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+
+"-TOTAL_FORMS");b=c("#"+a.prefix+"-empty");var f=b.clone(!0);f.removeClass(a.emptyCssClass).addClass(a.formCssClass).attr("id",a.prefix+"-"+l);f.is("tr")?f.children(":last").append('<div><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+"</a></div>"):f.is("ul")||f.is("ol")?f.append('<li><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+"</a></li>"):f.children(":first").append('<span><a class="'+a.deleteCssClass+'" href="javascript:void(0)">'+a.deleteText+
-"</a></span>");e.find("*").each(function(){k(this,a.prefix,d.val())});e.insertBefore(c(b));c(d).val(parseInt(d.val(),10)+1);l+=1;""!==g.val()&&0>=g.val()-d.val()&&h.parent().hide();e.find("a."+a.deleteCssClass).click(function(b){b.preventDefault();b=c(this).parents("."+a.formCssClass);b.remove();--l;a.removed&&a.removed(b);b=c("."+a.formCssClass);c("#id_"+a.prefix+"-TOTAL_FORMS").val(b.length);(""===g.val()||0<g.val()-b.length)&&h.parent().show();for(var d=function(){k(this,a.prefix,e)},e=0,f=b.length;e<
+"</a></span>");f.find("*").each(function(){k(this,a.prefix,d.val())});f.insertBefore(c(b));c(d).val(parseInt(d.val(),10)+1);l+=1;""!==g.val()&&0>=g.val()-d.val()&&h.parent().hide();f.find("a."+a.deleteCssClass).click(function(b){b.preventDefault();b=c(this).parents("."+a.formCssClass);b.remove();--l;a.removed&&a.removed(b);b=c("."+a.formCssClass);c("#id_"+a.prefix+"-TOTAL_FORMS").val(b.length);(""===g.val()||0<g.val()-b.length)&&h.parent().show();var d,f,e=function(){k(this,a.prefix,d)};d=0;for(f=
-f;e++)k(c(b).get(e),a.prefix,e),c(b.get(e)).find("*").each(d)});a.added&&a.added(e)})}return this};c.fn.formset.defaults={prefix:"form",addText:"add another",deleteText:"remove",addCssClass:"add-row",deleteCssClass:"delete-row",emptyCssClass:"empty-row",formCssClass:"dynamic-form",added:null,removed:null};c.fn.tabularFormset=function(b){var a=c(this),d=function(b){c(a.selector).not(".add-row").removeClass("row1 row2").filter(":even").addClass("row1").end().filter(":odd").addClass("row2")},k=function(){"undefined"!=
+b.length;d<f;d++)k(c(b).get(d),a.prefix,d),c(b.get(d)).find("*").each(e)});a.added&&a.added(f)})}return this};c.fn.formset.defaults={prefix:"form",addText:"add another",deleteText:"remove",addCssClass:"add-row",deleteCssClass:"delete-row",emptyCssClass:"empty-row",formCssClass:"dynamic-form",added:null,removed:null};c.fn.tabularFormset=function(b){var a=c(this),d=function(b){c(a.selector).not(".add-row").removeClass("row1 row2").filter(":even").addClass("row1").end().filter(":odd").addClass("row2")},
-typeof SelectFilter&&(c(".selectfilter").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!1)}),c(".selectfilterstacked").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!0)}))},f=function(a){a.find(".prepopulated_field").each(function(){var b=c(this).find("input, select, textarea"),d=b.data("dependency_list")||[],f=[];c.each(d,function(c,b){f.push("#"+a.find(".field-"+b).find("input, select, textarea").attr("id"))});f.length&&b.prepopulate(f,
+k=function(){"undefined"!==typeof SelectFilter&&(c(".selectfilter").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!1)}),c(".selectfilterstacked").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!0)}))},e=function(a){a.find(".prepopulated_field").each(function(){var b=c(this).find("input, select, textarea"),d=b.data("dependency_list")||[],e=[];c.each(d,function(c,b){e.push("#"+a.find(".field-"+b).find("input, select, textarea").attr("id"))});
-b.attr("maxlength"))})};a.formset({prefix:b.prefix,addText:b.addText,formCssClass:"dynamic-"+b.prefix,deleteCssClass:"inline-deletelink",deleteText:b.deleteText,emptyCssClass:"empty-form",removed:d,added:function(a){f(a);"undefined"!=typeof DateTimeShortcuts&&(c(".datetimeshortcuts").remove(),DateTimeShortcuts.init());k();d(a)}});return a};c.fn.stackedFormset=function(b){var a=c(this),d=function(b){c(a.selector).find(".inline_label").each(function(a){a+=1;c(this).html(c(this).html().replace(/(#\d+)/g,
+e.length&&b.prepopulate(e,b.attr("maxlength"))})};a.formset({prefix:b.prefix,addText:b.addText,formCssClass:"dynamic-"+b.prefix,deleteCssClass:"inline-deletelink",deleteText:b.deleteText,emptyCssClass:"empty-form",removed:d,added:function(a){e(a);"undefined"!==typeof DateTimeShortcuts&&(c(".datetimeshortcuts").remove(),DateTimeShortcuts.init());k();d(a)}});return a};c.fn.stackedFormset=function(b){var a=c(this),d=function(b){c(a.selector).find(".inline_label").each(function(a){a+=1;c(this).html(c(this).html().replace(/(#\d+)/g,
-"#"+a))})},k=function(){"undefined"!=typeof SelectFilter&&(c(".selectfilter").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!1)}),c(".selectfilterstacked").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!0)}))},f=function(a){a.find(".prepopulated_field").each(function(){var b=c(this).find("input, select, textarea"),d=b.data("dependency_list")||[],f=[];c.each(d,function(b,c){f.push("#"+a.find(".form-row .field-"+c).find("input, select, textarea").attr("id"))});
+"#"+a))})},k=function(){"undefined"!==typeof SelectFilter&&(c(".selectfilter").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!1)}),c(".selectfilterstacked").each(function(a,c){var b=c.name.split("-");SelectFilter.init(c.id,b[b.length-1],!0)}))},e=function(a){a.find(".prepopulated_field").each(function(){var b=c(this).find("input, select, textarea"),d=b.data("dependency_list")||[],e=[];c.each(d,function(b,c){e.push("#"+a.find(".form-row .field-"+c).find("input, select, textarea").attr("id"))});
-f.length&&b.prepopulate(f,b.attr("maxlength"))})};a.formset({prefix:b.prefix,addText:b.addText,formCssClass:"dynamic-"+b.prefix,deleteCssClass:"inline-deletelink",deleteText:b.deleteText,emptyCssClass:"empty-form",removed:d,added:function(a){f(a);"undefined"!=typeof DateTimeShortcuts&&(c(".datetimeshortcuts").remove(),DateTimeShortcuts.init());k();d(a)}});return a}})(django.jQuery);
+e.length&&b.prepopulate(e,b.attr("maxlength"))})};a.formset({prefix:b.prefix,addText:b.addText,formCssClass:"dynamic-"+b.prefix,deleteCssClass:"inline-deletelink",deleteText:b.deleteText,emptyCssClass:"empty-form",removed:d,added:function(a){e(a);"undefined"!==typeof DateTimeShortcuts&&(c(".datetimeshortcuts").remove(),DateTimeShortcuts.init());k();d(a)}});return a}})(django.jQuery);

django/contrib/admin/static/admin/js/prepopulate.js

@@ -1,12 +1,13 @@
 /*global URLify*/
 (function($) {
-    $.fn.prepopulate = function(dependencies, maxLength) {
+    $.fn.prepopulate = function(dependencies, maxLength, allowUnicode) {
         /*
             Depends on urlify.js
             Populates a selected field with the values of the dependent fields,
             URLifies and shortens the string.
             dependencies - array of dependent fields ids
             maxLength - maximum length of the URLify'd string
+            allowUnicode - Unicode support of the URLify'd string
         */
         return this.each(function() {
             var prepopulatedField = $(this);
@@ -24,7 +25,7 @@
                         values.push(field.val());
                     }
                 });
-                prepopulatedField.val(URLify(values.join(' '), maxLength));
+                prepopulatedField.val(URLify(values.join(' '), maxLength, allowUnicode));
             };
 
             prepopulatedField.data('_changed', false);

django/contrib/admin/static/admin/js/prepopulate.min.js

@@ -1 +1 @@
-(function(c){c.fn.prepopulate=function(e,f){return this.each(function(){var a=c(this),b=function(){if(!a.data("_changed")){var b=[];c.each(e,function(a,d){d=c(d);0<d.val().length&&b.push(d.val())});a.val(URLify(b.join(" "),f))}};a.data("_changed",!1);a.change(function(){a.data("_changed",!0)});a.val()||c(e.join(",")).keyup(b).change(b).focus(b)})}})(django.jQuery);
+(function(c){c.fn.prepopulate=function(e,f,g){return this.each(function(){var a=c(this),b=function(){if(!a.data("_changed")){var b=[];c.each(e,function(a,d){d=c(d);0<d.val().length&&b.push(d.val())});a.val(URLify(b.join(" "),f,g))}};a.data("_changed",!1);a.change(function(){a.data("_changed",!0)});a.val()||c(e.join(",")).keyup(b).change(b).focus(b)})}})(django.jQuery);

django/contrib/admin/static/admin/js/urlify.js

@@ -1,3 +1,4 @@
+/*global XRegExp*/
 var LATIN_MAP = {
     'À': 'A', 'Á': 'A', 'Â': 'A', 'Ã': 'A', 'Ä': 'A', 'Å': 'A', 'Æ': 'AE', 'Ç':
     'C', 'È': 'E', 'É': 'E', 'Ê': 'E', 'Ë': 'E', 'Ì': 'I', 'Í': 'I', 'Î': 'I',
@@ -132,10 +133,12 @@ function downcode(slug) {
 }
 
 
-function URLify(s, num_chars) {
+function URLify(s, num_chars, allowUnicode) {
     // changes, e.g., "Petty theft" to "petty_theft"
     // remove all these words from the string before urlifying
+    if (!allowUnicode) {
         s = downcode(s);
+    }
     var removelist = [
         "a", "an", "as", "at", "before", "but", "by", "for", "from", "is",
         "in", "into", "like", "of", "off", "on", "onto", "per", "since",
@@ -144,7 +147,13 @@ function URLify(s, num_chars) {
     var r = new RegExp('\\b(' + removelist.join('|') + ')\\b', 'gi');
     s = s.replace(r, '');
     // if downcode doesn't hit, the char will be stripped here
+    if (allowUnicode) {
+        // Keep Unicode letters including both lowercase and uppercase
+        // characters, whitespace, and dash; remove other characters.
+        s = XRegExp.replace(s, XRegExp('[^-_\\p{L}\\p{N}\\s]', 'g'), '');
+    } else {
         s = s.replace(/[^-\w\s]/g, '');  // remove unneeded chars
+    }
     s = s.replace(/^\s+|\s+$/g, ''); // trim leading/trailing spaces
     s = s.replace(/[-\s]+/g, '-');   // convert spaces to hyphens
     s = s.toLowerCase();             // convert to lowercase

django/contrib/admin/static/admin/js/vendor/xregexp/LICENSE-XREGEXP.txt

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2007-2012 Steven Levithan <http://xregexp.com/>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

django/contrib/admin/templates/admin/prepopulated_fields_js.html

@@ -8,7 +8,8 @@
         id: '#{{ field.field.auto_id }}',
         dependency_ids: [],
         dependency_list: [],
-        maxLength: {{ field.field.field.max_length|default_if_none:"50"|unlocalize }}
+        maxLength: {{ field.field.field.max_length|default:"50"|unlocalize }},
+        allowUnicode: {{ field.field.field.allow_unicode|default:"false"|lower }}
     };
 
     {% for dependency in field.dependencies %}
@@ -21,7 +22,7 @@
     {% endcomment %}
     $('.empty-form .form-row .field-{{ field.field.name }}, .empty-form.form-row .field-{{ field.field.name }}').addClass('prepopulated_field');
     $(field.id).data('dependency_list', field['dependency_list'])
-               .prepopulate(field['dependency_ids'], field.maxLength);
+               .prepopulate(field['dependency_ids'], field.maxLength, field.allowUnicode);
 {% endfor %}
 })(django.jQuery);
 </script>

django/core/validators.py

@@ -215,6 +215,13 @@ validate_slug = RegexValidator(
     'invalid'
 )
 
+slug_unicode_re = re.compile(r'^[-\w]+\Z', re.U)
+validate_unicode_slug = RegexValidator(
+    slug_unicode_re,
+    _("Enter a valid 'slug' consisting of Unicode letters, numbers, underscores, or hyphens."),
+    'invalid'
+)
+
 ipv4_re = re.compile(r'^(25[0-5]|2[0-4]\d|[0-1]?\d?\d)(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\Z')
 validate_ipv4_address = RegexValidator(ipv4_re, _('Enter a valid IPv4 address.'), 'invalid')
 

django/db/models/fields/__init__.py

@@ -2117,6 +2117,9 @@ class SlugField(CharField):
         # Set db_index=True unless it's been set manually.
         if 'db_index' not in kwargs:
             kwargs['db_index'] = True
+        self.allow_unicode = kwargs.pop('allow_unicode', False)
+        if self.allow_unicode:
+            self.default_validators = [validators.validate_unicode_slug]
         super(SlugField, self).__init__(*args, **kwargs)
 
     def deconstruct(self):
@@ -2127,13 +2130,15 @@ class SlugField(CharField):
             kwargs['db_index'] = False
         else:
             del kwargs['db_index']
+        if self.allow_unicode is not False:
+            kwargs['allow_unicode'] = self.allow_unicode
         return name, path, args, kwargs
 
     def get_internal_type(self):
         return "SlugField"
 
     def formfield(self, **kwargs):
-        defaults = {'form_class': forms.SlugField}
+        defaults = {'form_class': forms.SlugField, 'allow_unicode': self.allow_unicode}
         defaults.update(kwargs)
         return super(SlugField, self).formfield(**defaults)
 

django/forms/fields.py

@@ -1240,6 +1240,12 @@ class GenericIPAddressField(CharField):
 class SlugField(CharField):
     default_validators = [validators.validate_slug]
 
+    def __init__(self, *args, **kwargs):
+        self.allow_unicode = kwargs.pop('allow_unicode', False)
+        if self.allow_unicode:
+            self.default_validators = [validators.validate_unicode_slug]
+        super(SlugField, self).__init__(*args, **kwargs)
+
 
 class UUIDField(CharField):
     default_error_messages = {

django/utils/text.py

@@ -410,13 +410,17 @@ def unescape_string_literal(s):
 unescape_string_literal = allow_lazy(unescape_string_literal)
 
 
-def slugify(value):
+def slugify(value, allow_unicode=False):
     """
-    Converts to ASCII. Converts spaces to hyphens. Removes characters that
+    Convert to ASCII if 'allow_unicode' is False. Convert spaces to hyphens.
-    aren't alphanumerics, underscores, or hyphens. Converts to lowercase.
+    Remove characters that aren't alphanumerics, underscores, or hyphens.
-    Also strips leading and trailing whitespace.
+    Convert to lowercase. Also strip leading and trailing whitespace.
     """
     value = force_text(value)
+    if allow_unicode:
+        value = unicodedata.normalize('NFKC', value)
+        value = re.sub('[^\w\s-]', '', value, flags=re.U).strip().lower()
+        return mark_safe(re.sub('[-\s]+', '-', value, flags=re.U))
     value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
     value = re.sub('[^\w\s-]', '', value).strip().lower()
     return mark_safe(re.sub('[-\s]+', '-', value))

docs/ref/forms/fields.txt

@@ -875,6 +875,15 @@ For each field, we describe the default widget used if you don't specify
    This field is intended for use in representing a model
    :class:`~django.db.models.SlugField` in forms.
 
+   Takes an optional parameter:
+
+   .. attribute:: allow_unicode
+
+       .. versionadded:: 1.9
+
+       A boolean instructing the field to accept Unicode letters in addition
+       to ASCII letters. Defaults to ``False``.
+
 ``TimeField``
 ~~~~~~~~~~~~~
 

docs/ref/models/fields.txt

@@ -1012,6 +1012,13 @@ It is often useful to automatically prepopulate a SlugField based on the value
 of some other value.  You can do this automatically in the admin using
 :attr:`~django.contrib.admin.ModelAdmin.prepopulated_fields`.
 
+.. attribute:: SlugField.allow_unicode
+
+    .. versionadded:: 1.9
+
+    If ``True``, the field accepts Unicode letters in addition to ASCII
+    letters. Defaults to ``False``.
+
 ``SmallIntegerField``
 ---------------------
 

docs/ref/utils.txt

@@ -836,11 +836,11 @@ appropriate entities.
 .. module:: django.utils.text
     :synopsis: Text manipulation.
 
-.. function:: slugify
+.. function:: slugify(allow_unicode=False)
 
-    Converts to ASCII. Converts spaces to hyphens. Removes characters that
+    Converts to ASCII if ``allow_unicode`` is ``False`` (default). Converts spaces to
-    aren't alphanumerics, underscores, or hyphens. Converts to lowercase. Also
+    hyphens. Removes characters that aren't alphanumerics, underscores, or
-    strips leading and trailing whitespace.
+    hyphens. Converts to lowercase. Also strips leading and trailing whitespace.
 
     For example::
 
@@ -849,6 +849,17 @@ appropriate entities.
     If ``value`` is ``"Joel is a slug"``, the output will be
     ``"joel-is-a-slug"``.
 
+    You can set the ``allow_unicode`` parameter to ``True``, if you want to
+    allow Unicode characters::
+
+        slugify(value, allow_unicode=True)
+
+    If ``value`` is ``"你好 World"``, the output will be ``"你好-world"``.
+
+    .. versionchanged:: 1.9
+
+        The ``allow_unicode`` parameter was added.
+
 .. _time-zone-selection-functions:
 
 ``django.utils.timezone``

docs/ref/validators.txt

@@ -183,6 +183,16 @@ to, or in lieu of custom ``field.clean()`` methods.
     A :class:`RegexValidator` instance that ensures a value consists of only
     letters, numbers, underscores or hyphens.
 
+``validate_unicode_slug``
+-------------------------
+
+.. data:: validate_unicode_slug
+
+    .. versionadded:: 1.9
+
+    A :class:`RegexValidator` instance that ensures a value consists of only
+    Unicode letters, numbers, underscores, or hyphens.
+
 ``validate_ipv4_address``
 -------------------------
 

docs/releases/1.9.txt

@@ -308,6 +308,10 @@ Forms
 * You can now :ref:`specify keyword arguments <custom-formset-form-kwargs>`
   that you want to pass to the constructor of forms in a formset.
 
+* :class:`~django.forms.SlugField` now accepts an
+  :attr:`~django.forms.SlugField.allow_unicode` argument to allow Unicode
+  characters in slugs.
+
 * :class:`~django.forms.CharField` now accepts a
   :attr:`~django.forms.CharField.strip` argument to strip input data of leading
   and trailing whitespace.  As this defaults to ``True`` this is different
@@ -426,6 +430,10 @@ Models
 * Added the :class:`~django.db.models.functions.Now` database function, which
   returns the current date and time.
 
+* :class:`~django.db.models.SlugField` now accepts an
+  :attr:`~django.db.models.SlugField.allow_unicode` argument to allow Unicode
+  characters in slugs.
+
 CSRF
 ^^^^
 
@@ -536,6 +544,9 @@ Validators
 * :class:`~django.core.validators.EmailValidator` now limits the length of
   domain name labels to 63 characters per :rfc:`1034`.
 
+* Added :func:`~django.core.validators.validate_unicode_slug` to validate slugs
+  that may contain Unicode characters.
+
 Backwards incompatible changes in 1.9
 =====================================
 

tests/admin_views/admin.py

@@ -673,12 +673,15 @@ class MainPrepopulatedAdmin(admin.ModelAdmin):
     inlines = [RelatedPrepopulatedInline1, RelatedPrepopulatedInline2]
     fieldsets = (
         (None, {
-            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2',),)
+            'fields': (('pubdate', 'status'), ('name', 'slug1', 'slug2', 'slug3'))
         }),
     )
     formfield_overrides = {models.CharField: {'strip': False}}
-    prepopulated_fields = {'slug1': ['name', 'pubdate'],
+    prepopulated_fields = {
-                           'slug2': ['status', 'name']}
+        'slug1': ['name', 'pubdate'],
+        'slug2': ['status', 'name'],
+        'slug3': ['name'],
+    }
 
 
 class UnorderedObjectAdmin(admin.ModelAdmin):

tests/admin_views/models.py

@@ -716,6 +716,7 @@ class MainPrepopulated(models.Model):
                  ('option two', 'Option Two')))
     slug1 = models.SlugField(blank=True)
     slug2 = models.SlugField(blank=True)
+    slug3 = models.SlugField(blank=True, allow_unicode=True)
 
 
 class RelatedPrepopulated(models.Model):

tests/admin_views/tests.py

@@ -4412,11 +4412,13 @@ class SeleniumAdminViewsFirefoxTests(AdminSeleniumWebDriverTestCase):
         # Main form ----------------------------------------------------------
         self.selenium.find_element_by_css_selector('#id_pubdate').send_keys('2012-02-18')
         self.get_select_option('#id_status', 'option two').click()
-        self.selenium.find_element_by_css_selector('#id_name').send_keys(' this is the mAin nÀMë and it\'s awεšome')
+        self.selenium.find_element_by_css_selector('#id_name').send_keys(' this is the mAin nÀMë and it\'s awεšomeııı')
         slug1 = self.selenium.find_element_by_css_selector('#id_slug1').get_attribute('value')
         slug2 = self.selenium.find_element_by_css_selector('#id_slug2').get_attribute('value')
-        self.assertEqual(slug1, 'main-name-and-its-awesome-2012-02-18')
+        slug3 = self.selenium.find_element_by_css_selector('#id_slug3').get_attribute('value')
-        self.assertEqual(slug2, 'option-two-main-name-and-its-awesome')
+        self.assertEqual(slug1, 'main-name-and-its-awesomeiii-2012-02-18')
+        self.assertEqual(slug2, 'option-two-main-name-and-its-awesomeiii')
+        self.assertEqual(slug3, 'main-n\xe0m\xeb-and-its-aw\u03b5\u0161ome\u0131\u0131\u0131')
 
         # Stacked inlines ----------------------------------------------------
         # Initial inline
@@ -4463,11 +4465,11 @@ class SeleniumAdminViewsFirefoxTests(AdminSeleniumWebDriverTestCase):
         self.wait_page_loaded()
         self.assertEqual(MainPrepopulated.objects.all().count(), 1)
         MainPrepopulated.objects.get(
-            name=' this is the mAin nÀMë and it\'s awεšome',
+            name=' this is the mAin nÀMë and it\'s awεšomeııı',
             pubdate='2012-02-18',
             status='option two',
-            slug1='main-name-and-its-awesome-2012-02-18',
+            slug1='main-name-and-its-awesomeiii-2012-02-18',
-            slug2='option-two-main-name-and-its-awesome',
+            slug2='option-two-main-name-and-its-awesomeiii',
         )
         self.assertEqual(RelatedPrepopulated.objects.all().count(), 4)
         RelatedPrepopulated.objects.get(

tests/forms_tests/tests/test_fields.py

@@ -1556,6 +1556,16 @@ class FieldsTests(SimpleTestCase):
         f = SlugField()
         self.assertEqual(f.clean('    aa-bb-cc    '), 'aa-bb-cc')
 
+    def test_slugfield_unicode_normalization(self):
+        f = SlugField(allow_unicode=True)
+        self.assertEqual(f.clean('a'), 'a')
+        self.assertEqual(f.clean('1'), '1')
+        self.assertEqual(f.clean('a1'), 'a1')
+        self.assertEqual(f.clean('你好'), '你好')
+        self.assertEqual(f.clean('  你-好  '), '你-好')
+        self.assertEqual(f.clean('ıçğüş'), 'ıçğüş')
+        self.assertEqual(f.clean('foo-ıç-bar'), 'foo-ıç-bar')
+
     # UUIDField ###################################################################
 
     def test_uuidfield_1(self):

tests/model_fields/models.py

@@ -86,6 +86,10 @@ class BigS(models.Model):
     s = models.SlugField(max_length=255)
 
 
+class UnicodeSlugField(models.Model):
+    s = models.SlugField(max_length=255, allow_unicode=True)
+
+
 class SmallIntegerModel(models.Model):
     value = models.SmallIntegerField()
 

tests/model_fields/tests.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 import datetime
@@ -20,6 +21,7 @@ from django.db.models.fields import (
 from django.db.models.fields.files import FileField, ImageField
 from django.test.utils import requires_tz_support
 from django.utils import six, timezone
+from django.utils.encoding import force_str
 from django.utils.functional import lazy
 
 from .models import (
@@ -27,7 +29,8 @@ from .models import (
     Document, FksToBooleans, FkToChar, FloatModel, Foo, GenericIPAddress,
     IntegerModel, NullBooleanModel, PositiveIntegerModel,
     PositiveSmallIntegerModel, Post, PrimaryKeyCharModel, RenamedField,
-    SmallIntegerModel, VerboseNameField, Whiz, WhizIter, WhizIterEmpty,
+    SmallIntegerModel, UnicodeSlugField, VerboseNameField, Whiz, WhizIter,
+    WhizIterEmpty,
 )
 
 
@@ -113,7 +116,6 @@ class BasicFieldTests(test.TestCase):
         self.assertIsInstance(field.formfield(choices_form_class=klass), klass)
 
     def test_field_str(self):
-        from django.utils.encoding import force_str
         f = Foo._meta.get_field('a')
         self.assertEqual(force_str(f), "model_fields.Foo.a")
 
@@ -515,6 +517,14 @@ class SlugFieldTests(test.TestCase):
         bs = BigS.objects.get(pk=bs.pk)
         self.assertEqual(bs.s, 'slug' * 50)
 
+    def test_slugfield_unicode_max_length(self):
+        """
+        SlugField with allow_unicode=True should honor max_length.
+        """
+        bs = UnicodeSlugField.objects.create(s='你好你好' * 50)
+        bs = UnicodeSlugField.objects.get(pk=bs.pk)
+        self.assertEqual(bs.s, '你好你好' * 50)
+
 
 class ValidationTest(test.SimpleTestCase):
     def test_charfield_raises_error_on_empty_string(self):

tests/utils_tests/test_text.py

@@ -172,11 +172,16 @@ class TestUtilsText(SimpleTestCase):
 
     def test_slugify(self):
         items = (
-            ('Hello, World!', 'hello-world'),
+            # given - expected - unicode?
-            ('spam & eggs', 'spam-eggs'),
+            ('Hello, World!', 'hello-world', False),
+            ('spam & eggs', 'spam-eggs', False),
+            ('spam & ıçüş', 'spam-ıçüş', True),
+            ('foo ıç bar', 'foo-ıç-bar', True),
+            ('    foo ıç bar', 'foo-ıç-bar', True),
+            ('你好', '你好', True),
         )
-        for value, output in items:
+        for value, output, is_unicode in items:
-            self.assertEqual(text.slugify(value), output)
+            self.assertEqual(text.slugify(value, allow_unicode=is_unicode), output)
 
     def test_unescape_entities(self):
         items = [

tests/validators/tests.py

@@ -14,7 +14,7 @@ from django.core.validators import (
     MinLengthValidator, MinValueValidator, RegexValidator, URLValidator,
     int_list_validator, validate_comma_separated_integer_list, validate_email,
     validate_integer, validate_ipv4_address, validate_ipv6_address,
-    validate_ipv46_address, validate_slug,
+    validate_ipv46_address, validate_slug, validate_unicode_slug,
 )
 from django.test import SimpleTestCase
 from django.test.utils import str_prefix
@@ -89,15 +89,36 @@ TEST_DATA = [
     (validate_slug, 'longer-slug-still-ok', None),
     (validate_slug, '--------', None),
     (validate_slug, 'nohyphensoranything', None),
+    (validate_slug, 'a', None),
+    (validate_slug, '1', None),
+    (validate_slug, 'a1', None),
 
     (validate_slug, '', ValidationError),
     (validate_slug, ' text ', ValidationError),
     (validate_slug, ' ', ValidationError),
     (validate_slug, 'some@mail.com', ValidationError),
     (validate_slug, '你好', ValidationError),
+    (validate_slug, '你 好', ValidationError),
     (validate_slug, '\n', ValidationError),
     (validate_slug, 'trailing-newline\n', ValidationError),
 
+    (validate_unicode_slug, 'slug-ok', None),
+    (validate_unicode_slug, 'longer-slug-still-ok', None),
+    (validate_unicode_slug, '--------', None),
+    (validate_unicode_slug, 'nohyphensoranything', None),
+    (validate_unicode_slug, 'a', None),
+    (validate_unicode_slug, '1', None),
+    (validate_unicode_slug, 'a1', None),
+    (validate_unicode_slug, '你好', None),
+
+    (validate_unicode_slug, '', ValidationError),
+    (validate_unicode_slug, ' text ', ValidationError),
+    (validate_unicode_slug, ' ', ValidationError),
+    (validate_unicode_slug, 'some@mail.com', ValidationError),
+    (validate_unicode_slug, '\n', ValidationError),
+    (validate_unicode_slug, '你 好', ValidationError),
+    (validate_unicode_slug, 'trailing-newline\n', ValidationError),
+
     (validate_ipv4_address, '1.1.1.1', None),
     (validate_ipv4_address, '255.0.0.0', None),
     (validate_ipv4_address, '0.0.0.0', None),