* -CSD does not affect modules, so readfile() was not using the utf-8 input

  layer, which led to lots of problems; make it force read files as utf-8.
  Closes: #373203
* writefile() likewise needs to use the utf8 output layer.
* Remove the -CSD from ikiwiki's hashbang since it's useless to have it
  there.
* Revert some of the decode_utf8 changes in CGI.pm that seem unnecessary
  given the readfile fix.
* Add utf-8 testcases for readfile and htmlize.

3 files changed, 33 insertions, 0 deletions

diff --git a/t/htmlize.t b/t/htmlize.t
new file mode 100755
index 000000000..465004bc1
--- /dev/null
+++ b/t/htmlize.t
@@ -0,0 +1,19 @@
+#!/usr/bin/perl
+use warnings;
+use strict;
+use Test::More tests => 4;
+use Encode;
+
+BEGIN { use_ok("IkiWiki"); }
+BEGIN { use_ok("IkiWiki::Render"); }
+
+# Initialize htmlscrubber plugin
+%IkiWiki::config=IkiWiki::defaultconfig();
+$IkiWiki::config{srcdir}=$IkiWiki::config{destdir}="/dev/null";
+IkiWiki::checkconfig();
+
+is(IkiWiki::htmlize(".mdwn", "foo\n\nbar\n"), "<p>foo</p>\n\n<p>bar</p>\n",
+	"basic");
+is(IkiWiki::htmlize(".mdwn", IkiWiki::readfile("t/test1.mdwn")),
+	Encode::decode_utf8(qq{<p><img src="../images/o.jpg" alt="o" title="&oacute;" />\nóóóóó</p>\n}),
+	"utf8; bug #373203");
diff --git a/t/readfile.t b/t/readfile.t
new file mode 100755
index 000000000..5332f4c35
--- /dev/null
+++ b/t/readfile.t
@@ -0,0 +1,12 @@
+#!/usr/bin/perl
+use warnings;
+use strict;
+use Test::More tests => 3;
+use Encode;
+
+BEGIN { use_ok("IkiWiki"); }
+
+# should read files as utf8
+ok(Encode::is_utf8(IkiWiki::readfile("t/test1.mdwn"), 1));
+is(IkiWiki::readfile("t/test1.mdwn"),
+	Encode::decode_utf8('![o](../images/o.jpg "ó")'."\n".'óóóóó'."\n"));
diff --git a/t/test1.mdwn b/t/test1.mdwn
new file mode 100644
index 000000000..f4ebc2c08
--- /dev/null
+++ b/t/test1.mdwn
@@ -0,0 +1,2 @@
+![o](../images/o.jpg "ó")
+óóóóó