Implemented --underlaydir, and moved files provided by underlay out of doc

so I don't need to maintain two copies anymore.

You might also want to remove the files provided in the basewiki underlay
from your wiki, if you have not created custom local versions of them, so
that these pages will be automatically updated in future ikiwiki upgrades.

19 files changed, 97 insertions, 268 deletions

diff --git a/IkiWiki/CGI.pm b/IkiWiki/CGI.pm
index 67bce6795..6fd1f6506 100644
--- a/IkiWiki/CGI.pm
+++ b/IkiWiki/CGI.pm
@@ -403,7 +403,7 @@ sub cgi_editpage ($$) { #{{{
 			    ! length $form->field('content')) {
 				my $content="";
 				if (exists $pagesources{lc($page)}) {
-					$content=readfile("$config{srcdir}/$pagesources{lc($page)}");
+					$content=readfile(srcfile($pagesources{lc($page)}));
 					$content=~s/\n/\r\n/g;
 				}
 				$form->field(name => "content", value => $content,
diff --git a/IkiWiki/Rcs/SVN.pm b/IkiWiki/Rcs/SVN.pm
index 083b869df..02fc3ed31 100644
--- a/IkiWiki/Rcs/SVN.pm
+++ b/IkiWiki/Rcs/SVN.pm
@@ -171,6 +171,7 @@ sub rcs_getctime () { #{{{
 	eval q{use Date::Parse};
 	foreach my $page (keys %pagectime) {
 		my $file="$config{srcdir}/$pagesources{$page}";
+		next unless -e $file;
 		my $child = open(SVNLOG, "-|");
 		if (! $child) {
 			exec("svn", "log", $file) || error("svn log $file failed to run");
diff --git a/IkiWiki/Render.pm b/IkiWiki/Render.pm
index 7d1e8ee53..3d827d341 100644
--- a/IkiWiki/Render.pm
+++ b/IkiWiki/Render.pm
@@ -139,7 +139,7 @@ sub get_inline_content ($$) { #{{{
 	my $file=$pagesources{$page};
 	my $type=pagetype($file);
 	if ($type ne 'unknown') {
-		return htmlize($type, linkify(readfile("$config{srcdir}/$file"), $parentpage));
+		return htmlize($type, linkify(readfile(srcfile($file)), $parentpage));
 	}
 	else {
 		return "";
@@ -337,7 +337,8 @@ sub render ($) { #{{{
 	my $file=shift;
 	
 	my $type=pagetype($file);
-	my $content=readfile("$config{srcdir}/$file");
+	my $srcfile=srcfile($file);
+	my $content=readfile($srcfile);
 	if ($type ne 'unknown') {
 		my $page=pagename($file);
 		
@@ -349,7 +350,7 @@ sub render ($) { #{{{
 		
 		check_overwrite("$config{destdir}/".htmlpage($page), $page);
 		writefile("$config{destdir}/".htmlpage($page),
-			genpage($content, $page, mtime("$config{srcdir}/$file")));
+			genpage($content, $page, mtime($srcfile)));
 		$oldpagemtime{$page}=time;
 		$renderedfiles{$page}=htmlpage($page);
 
@@ -358,7 +359,7 @@ sub render ($) { #{{{
 		# only supports listing one file per page.
 		if ($config{rss} && exists $inlinepages{$page}) {
 			writefile("$config{destdir}/".rsspage($page),
-				genrss($content, $page, mtime("$config{srcdir}/$file")));
+				genrss($content, $page, mtime($srcfile)));
 		}
 	}
 	else {
@@ -389,9 +390,7 @@ sub refresh () { #{{{
 		no_chdir => 1,
 		wanted => sub {
 			if (/$config{wiki_file_prune_regexp}/) {
-				no warnings 'once';
 				$File::Find::prune=1;
-				use warnings "all";
 			}
 			elsif (! -d $_ && ! -l $_) {
 				my ($f)=/$config{wiki_file_regexp}/; # untaint
@@ -406,6 +405,30 @@ sub refresh () { #{{{
 			}
 		},
 	}, $config{srcdir});
+	find({
+		no_chdir => 1,
+		wanted => sub {
+			if (/$config{wiki_file_prune_regexp}/) {
+				$File::Find::prune=1;
+			}
+			elsif (! -d $_ && ! -l $_) {
+				my ($f)=/$config{wiki_file_regexp}/; # untaint
+				if (! defined $f) {
+					warn("skipping bad filename $_\n");
+				}
+				else {
+					# Don't add files that are in the
+					# srcdir.
+					$f=~s/^\Q$config{underlaydir}\E\/?//;
+					if (! -e "$config{srcdir}/$f" && 
+					    ! -l "$config{srcdir}/$f") {
+						push @files, $f;
+						$exists{pagename($f)}=1;
+					}
+				}
+			}
+		},
+	}, $config{underlaydir});
 
 	my %rendered;
 
@@ -418,7 +441,7 @@ sub refresh () { #{{{
 			push @add, $file;
 			$links{$page}=[];
 			$pagesources{$page}=$file;
-			$pagectime{$page}=mtime("$config{srcdir}/$file") 
+			$pagectime{$page}=mtime(srcfile($file))
 				unless exists $pagectime{$page};
 		}
 	}
@@ -439,7 +462,7 @@ sub refresh () { #{{{
 		my $page=pagename($file);
 		
 		if (! exists $oldpagemtime{$page} ||
-		    mtime("$config{srcdir}/$file") > $oldpagemtime{$page}) {
+		    mtime(srcfile($file)) > $oldpagemtime{$page}) {
 		    	debug("rendering changed file $file");
 			render($file);
 			$rendered{$file}=1;
diff --git a/Makefile.PL b/Makefile.PL
index 10015c47c..f3daaed2f 100755
--- a/Makefile.PL
+++ b/Makefile.PL
@@ -12,7 +12,7 @@ install:: extra_install
 pure_install:: extra_install
 
 extra_build:
-	./ikiwiki doc html --templatedir=templates \
+	./ikiwiki doc html --templatedir=templates --underlaydir=basewiki \
 		--wikiname="ikiwiki" --verbose --nosvn --exclude=/discussion
 	./mdwn2man doc/usage.mdwn > ikiwiki.man
 		
diff --git a/doc/blog.mdwn b/doc/blog.mdwn
deleted file mode 100644
index 9c490fcb3..000000000
--- a/doc/blog.mdwn
+++ /dev/null
@@ -1,22 +0,0 @@
-You can turn any page on this wiki into a weblog by inserting a
-[[PostProcessorDirective]]. Like this:
-
-\\[[inline pages="blog/* !*/Discussion" show="10" rootpage="blog"]]
-
-Any pages that match the specified [[GlobList]] (in the example, any
-[[SubPage]] of "blog") will be part of the blog, and the newest 10
-of them will appear in the page.
-
-The optional `rootpage` setting tells the wiki that new posts to this blog
-should default to being [[SubPage]] of "blog", and enables a form at the
-top of the blog that can be used to add new items.
-
-If you want your blog to have an archive page listing every post ever made
-to it, you can accomplish that like this:
-
-\\[[inline pages="blog/* !*/Discussion" archive="yes"]]
-
-You can even create an automatically generated list of all the pages on the
-wiki, with the most recently added at the top, like this:
-
-\\[[inline pages="* !*/Discussion" archive="yes"]]
diff --git a/doc/bugs.mdwn b/doc/bugs.mdwn
index 8685511c5..d1c3bd4b9 100644
--- a/doc/bugs.mdwn
+++ b/doc/bugs.mdwn
@@ -29,3 +29,8 @@
   preview, it doesn't get the link right because it makes it relative to
   where the page will be saved to, not to where the cgi script is.
 * RSS output contains relative links. Ie. http://kitenet.net/~joey/blog/index.rss contains a link to http://kitenet.net/~joey/blog/../blog.html
+* If a file in the srcdir is removed, exposing a file in the underlaydir,
+  ikiwiki will not notice the change and rebuild it until the file in the
+  underlaydir gets a mtime newer than the mtime the removed file had.
+* Pages rendered from files in the underlaydir should not have a history
+  link, since there's no file in the svn repo to view the history of.
diff --git a/doc/globlist.mdwn b/doc/globlist.mdwn
deleted file mode 100644
index 30bc837ac..000000000
--- a/doc/globlist.mdwn
+++ /dev/null
@@ -1,16 +0,0 @@
-When the wiki stores lists of pages, such as pages that are locked or pages
-that you want to be emailed if changed, it uses a GlobList.
-
-This is a list of page names, separated by white space. The "glob" bit is
-that as well as full page names, it can contain glob patterns. "`*`" stands
-in for any part of the page name, and "`?`" for any single letter of its
-name. So if you wanted to list all the pages about tea, and any
-[[SubPage]]s of the SandBox, but not including the SandBox itself:
-
-	*tea* SandBox/*
-
-You can also prefix an item in the list with "`!`" to skip matching any
-pages that match it. So if you want to specify all pages except for
-Discussion pages and the SandBox:
-
-	* !SandBox !*/Discussion
diff --git a/doc/helponformatting.mdwn b/doc/helponformatting.mdwn
deleted file mode 100644
index 4ef41d16f..000000000
--- a/doc/helponformatting.mdwn
+++ /dev/null
@@ -1,59 +0,0 @@
-## Help on formatting text
-
-Text on this wiki is written in a form very close to how you might write
-text for an email message.
-
-Leave blank lines between paragraphs.
-
-You can \**emphasise*\* or \*\***strongly emphasise**\*\* text by placing it
-in single or double asterisks.
-
-To create a list, start each line with an asterisk:
-
-* "* this is my list"
-* "* another item"
-
-To make a numbered list, start each line with a number (any number will
-do) followed by a period:
-
-1. "1. first line"
-2. "2. second line"
-2. "2. third line"
-
-To create a header, start a line with one or more `#` characters followed
-by a space and the header text. The number of `#` characters controls the
-size of the header:
-
-## ## h2
-### ### h3
-#### #### h4
-
-To create a horizontal rule, just write three or more dashes on their own
-line:
-
-----
-
-To quote someone, prefix the quote with ">":
-
-> To be or not to be,
-> that is the question.
-
-To write a code block, indent each line with a tab:
-
-	10 PRINT "Hello, world!"
-	20 GOTO 10
-
-To link to another page on the wiki, place the page's name inside double
-square brackets, so you would use `\[[WikiLink]]` to link to [[WikiLink]].
-
-To link to any other web page, or to an email address, you can just put the url in angle brackets: <<http://ikiwiki.kitenet.net>>, or you can use the form
-\[link text\]\(url\)
-
-----
-
-Advanced users can use [[PostProcessorDirective]]s to do additional cool
-stuff.
-
-----
-
-This style of text formatting is called [[MarkDown]].
diff --git a/doc/markdown.mdwn b/doc/markdown.mdwn
deleted file mode 100644
index 3684fe5c1..000000000
--- a/doc/markdown.mdwn
+++ /dev/null
@@ -1,8 +0,0 @@
-[Markdown](http://daringfireball.net/projects/markdown/)
-is a minimal markup language that resembles plain text as used in
-email messages. It is the markup language used by this wiki.
-
-For documentation about the markdown syntax, see [[HelpOnFormatting]] and
-[Markdown: syntax](http://daringfireball.net/projects/markdown/syntax).
-
-Note that [[WikiLink]]s and [[PostProcessorDirective]]s are not part of the markdown syntax, and are the only bit of markup that this wiki handles internally.
diff --git a/doc/postprocessordirective.mdwn b/doc/postprocessordirective.mdwn
deleted file mode 100644
index fa8432e3f..000000000
--- a/doc/postprocessordirective.mdwn
+++ /dev/null
@@ -1,11 +0,0 @@
-Postprocessor directives are similar to a [[WikiLink]] in form, except they
-contain spaces and parameters. The general form is:
-
-\\[[directive param="value" param="value"]]
-
-This gets expanded after the rest of the page is processed, and can be used
-to transform the page in various ways.
-
-Currently, these postprocessor directives are available:
-
-* "inline" to make a [[blog]]
diff --git a/doc/security.mdwn b/doc/security.mdwn
index 48d82db89..0f8861d0d 100644
--- a/doc/security.mdwn
+++ b/doc/security.mdwn
@@ -162,3 +162,25 @@ again when saving the changed page.
 
 This was fixed by making ikiwiki refuse to read or write to files that are
 symlinks, combined with the above locking.
+
+## underlaydir override attacks
+
+ikiwiki also scans an underlaydir for pages, this is used to provide stock
+pages to all wikis w/o needing to copy them into the wiki. Since ikiwiki
+internally stores only the base filename from the underlaydir or srcdir,
+and searches for a file in either directory when reading a page source,
+there is the potential for ikiwiki's scanner to reject a file from the
+srcdir for some reason (such as it being a symlink), find a valid copy of
+the file in the underlaydir, and then when loading the file, mistekenly
+load the bad file from the srcdir.
+
+This attack is avoided by making ikiwiki scan the srcdir first, and refuse
+to add any files from the underlaydir if a file also exists in the srcdir
+with the same name. **But**, note that this assumes that any given page can
+be produced from a file with only one name (`page.mdwn` => `page.html`).
+
+If it's possible for files with different names to produce a given page, it
+would still be possible to use this attack to confuse ikiwiki into
+rendering the wrong thing. This is not currently possible, but must be kept
+in mind in the future when for example adding support for generating html
+pages from source with some other extension.
diff --git a/doc/setup.mdwn b/doc/setup.mdwn
index e9c690f45..b49c2918e 100644
--- a/doc/setup.mdwn
+++ b/doc/setup.mdwn
@@ -19,22 +19,24 @@ optional support for commits from the web.
 
 		svn co file:///svn/wikirepo/trunk ~/wikiwc
 
-4. Create some files and add them into subversion. Or you might copy the
-   files from /usr/share/ikiwiki/basewiki and check those in to get a
-   head start on creating your wiki.
-
-		echo "Welcome to my empty wiki." > ~/wikiwc/index.mdwn
-		echo "Feel free to edit this page" > ~/wikiwc/sandbox.mdwn
-		svn add ~/wikiwc/*.mdwn
-		svn commit ~/wikiwc -m add
-
-5. Build your wiki for the first time.
+4. Build your wiki for the first time.
 
 		ikiwiki --verbose ~/wikiwc/ ~/public_html/wiki/ \
 			--url=http://host/~you/wiki/
 
    Replace the url with the real url to your wiki. You should now
-   be able to visit the url and see your page that you created earlier.
+   be able to visit the url and see your wiki.
+
+5. Customise your wiki. The files in `/usr/share/ikiwiki/basewiki/` are
+   used if you don't have a custom version, so let's start by making a
+   custom version of the wiki's index page:
+
+   		cp /usr/share/ikiwiki/basewiki/index.mdwn ~/wikiwc
+		svn add ~/wikiwc/index.mdwn
+		$EDITOR ~/wikiwc/index.mdwn
+		svn commit ~/wikiwc/index.mdwn -m customised
+
+   You can also add any files you like from scratch of course.
 
 6. Repeat steps 4 and 5 as desired, editing or adding pages and rebuilding
    the wiki. You can play around with other ikiwiki parameters such as
diff --git a/doc/style.css b/doc/style.css
deleted file mode 100644
index 97b30fbf8..000000000
--- a/doc/style.css
+++ /dev/null
@@ -1,71 +0,0 @@
-#header h1 {
-	margin: 0;
-	padding: 2px 0;
-}
-
-#actions ul {
-	margin: 0;
-	padding: 2px;
-	list-style-type: none;
-	border-bottom: 1px solid #000;
-}
-
-#actions li {
-	display: inline;
-	padding: .2em .4em;
-}
-
-#content {
-	border-bottom: 1px solid #000;
-}
-
-/* Used for adding a blog page. */
-#blogform {
-	padding: 10px 10px;
-	border: 1px solid #aaa;
-	color: black !important;;
-	background: #eee;
-}
-
-#backlinks {
-	margin: 1em 0;
-}
-
-#footer {
-	margin: 1em 0;
-}
-
-#pageinfo {
-	font-style: italic;
-}
-
-/* Used for invalid form fields. */
-.fb_invalid {
-	color: red;
-	background: white !important;
-}
-
-/* Used for required form fields. */
-.fb_required {
-	font-weight: bold;
-}
-
-/* RSS button. */
-.rssbutton {
-	background: #ff6600;
-	color: white !important;
-	border-left: 1px solid #cc9966;
-	border-top: 1px solid #ccaa99;
-	border-right: 1px solid #993300;
-	border-bottom: 1px solid #331100;
-	padding: 0px 0.5em 0px 0.5em;
-	font-family: helvetica, arial, sans-serif;
-	font-weight: bold;
-	font-size: small;
-	text-decoration: none;
-	margin-top: 1em;
-}
-.rssbutton:hover {
-	color: white !important;
-	background: #ff9900;
-}
diff --git a/doc/subpage.mdwn b/doc/subpage.mdwn
deleted file mode 100644
index 43669209c..000000000
--- a/doc/subpage.mdwn
+++ /dev/null
@@ -1,11 +0,0 @@
-ikiwiki supports placing pages in a directory hierarchy. For example,
-this page, [[SubPage]] has some related pages placed under it, like
-[[SubPage/LinkingRules]]. This is a useful way to add some order to your
-wiki rather than just having a great big directory full of pages.
-
-To add a SubPage, just make a subdirectory and put pages in it. For
-example, this page is SubPage.mdwn in this wiki's source, and there is also
-a SubPage subdirectory, which contains SubPage/LinkingRules.mdwn. Subpages
-can be nested as deeply as you'd like.
-
-Linking to and from a SubPage is explained in [[LinkingRules]].
diff --git a/doc/subpage/linkingrules.mdwn b/doc/subpage/linkingrules.mdwn
deleted file mode 100644
index 83625ccbd..000000000
--- a/doc/subpage/linkingrules.mdwn
+++ /dev/null
@@ -1,21 +0,0 @@
-To link to or from a [[SubPage]], you can normally use a regular
-[[WikiLink]] that does not contain the name of the parent directory of
-the [[SubPage]]. Ikiwiki descends the directory hierarchy looking for a
-page that matches your link.
-
-For example, if FooBar/SubPage links to "OtherPage", ikiwiki will first 
-prefer pointing the link to FooBar/SubPage/OtherPage if it exists, next
-to FooBar/OtherPage and finally to OtherPage in the root of the wiki.
-
-Note that this means that if a link on FooBar/SomePage to "OtherPage"
-currently links to OtherPage, in the root of the wiki, and FooBar/OtherPage
-is created, the link will _change_ to point to FooBar/OtherPage. On the
-other hand, a link from BazBar to "OtherPage" would be unchanged by this
-creation of a [[SubPage]] of FooBar.
-
-You can also specify a link that contains a directory name, like
-"FooBar/OtherPage" to more exactly specify what page to link to. This is
-the only way to link to an unrelated [[SubPage]].
-
-You can use this to, for example, to link from BazBar to "FooBar/SubPage",
-or from BazBar/SubPage to "FooBar/SubPage".
diff --git a/doc/todo/underlay.mdwn b/doc/todo/done/underlay.mdwn
index 20266260f..48e79498d 100644
--- a/doc/todo/underlay.mdwn
+++ b/doc/todo/done/underlay.mdwn
@@ -2,14 +2,10 @@ Rather than copy the basewiki around everywhere, it should be configured to
 underlay the main srcdir, and pages be rendered from there if not in the
 srcdir. This would allow upgrades to add/edit pages in the basewiki.
 
-Impementaion will be slightly tricky since currently ikiwiki is hardcoded
+Implementaion will be slightly tricky since currently ikiwiki is hardcoded
 in many places to look in srcdir for pages. Also, there are possible
 security attacks in the vein of providing a file ikiwiki would normally
 skip in the srcdir, and tricking it to processing this file instead of the
-one from the underlaydir.
-
-There are also difficulties related to removing files from the srcdir, and
-exposing ones from the underlaydir. Will need to make sure that the mtime
-for the source file is zeroed when the page is removed, and that it then
-finds the underlay file and treats it as newer.
-
+one from the underlaydir. -- Fixed by scanning srcdir first, then
+underlaydir, and refusing to add any files from underlaydir if they also
+exist in the srcdir. However, see [[security]] for caveats.
diff --git a/doc/usage.mdwn b/doc/usage.mdwn
index f477fe562..9a8b97ceb 100644
--- a/doc/usage.mdwn
+++ b/doc/usage.mdwn
@@ -82,6 +82,12 @@ These options configure the wiki.
   Specify the directory that the page [[templates]] are stored in.
   Default is `/usr/share/ikiwiki/templates`.
 
+* --underlaydir
+
+  Specify the directory that is used to underlay the source directory.
+  Source files will be taken from here unless overridden by a file in the
+  source directory. Default is `/usr/share/ikiwiki/basewiki`.
+
 * --wrappermode mode
 
   Specify a mode to chmod the wrapper to after creating it.
diff --git a/doc/wikilink.mdwn b/doc/wikilink.mdwn
deleted file mode 100644
index 6051fe120..000000000
--- a/doc/wikilink.mdwn
+++ /dev/null
@@ -1,19 +0,0 @@
-WikiLinks provide easy linking between pages of the wiki. To create a
-[[WikiLink]], just put the name of the page to link to in double brackets.
-For example "\[[WikiLink]]".
-
-If you ever need to write something like "\[[WikiLink]] without creating a
-wikilink, just prefix it with a "\", like "\\\\[[WikiLink]]".
-
-Note that there are some special [[SubPage/LinkingRules]] that come into
-play when linking between [[SubPage]]s.
-
-WikiLinks can be entered in any case you like, the page they link to is
-always lowercased.
-
-Note that if the file linked to by a WikiLink looks like an image, it will
-be displayed inline on the page.
-
-It's also possible to write a WikiLink that uses something other than the
-page name as the link text. For example "\[[foo|SandBox]]" links to the
-SandBox page, but the link will appear like this: [[foo|SandBox]]
diff --git a/ikiwiki b/ikiwiki
index 7a16be3ca..4ef6ceba3 100755
--- a/ikiwiki
+++ b/ikiwiki
@@ -18,7 +18,7 @@ sub usage () { #{{{
 sub getconfig () { #{{{
 	if (! exists $ENV{WRAPPED_OPTIONS}) {
 		%config=(
-			wiki_file_prune_regexp => qr{((^|/).svn/|\.\.|^\.|\/\.|\.html?$)},
+			wiki_file_prune_regexp => qr{((^|/).svn/|\.\.|^\.|\/\.|\.html?$|\.rss$)},
 			wiki_link_regexp => qr/\[\[(?:([^\s\]\|]+)\|)?([^\s\]]+)\]\]/,
 			wiki_processor_regexp => qr/\[\[(\w+)\s+([^\]]+)\]\]/,
 			wiki_file_regexp => qr/(^[-A-Za-z0-9_.:\/+]+$)/,
@@ -41,6 +41,7 @@ sub getconfig () { #{{{
 			srcdir => undef,
 			destdir => undef,
 			templatedir => "/usr/share/ikiwiki/templates",
+			underlaydir => "/usr/share/ikiwiki/basewiki",
 			setup => undef,
 			adminuser => undef,
 		);
@@ -71,6 +72,9 @@ sub getconfig () { #{{{
 			"templatedir=s" => sub {
 				$config{templatedir}=possibly_foolish_untaint($_[1])
 			},
+			"underlaydir=s" => sub {
+				$config{underlaydir}=possibly_foolish_untaint($_[1])
+			},
 			"wrapper:s" => sub {
 				$config{wrapper}=$_[1] ? $_[1] : "ikiwiki-wrap"
 			},
@@ -176,6 +180,14 @@ sub htmlpage ($) { #{{{
 	return $page.".html";
 } #}}}
 
+sub srcfile ($) { #{{{
+	my $file=shift;
+
+	return "$config{srcdir}/$file" if -e "$config{srcdir}/$file";
+	return "$config{underlaydir}/$file" if -e "$config{underlaydir}/$file";
+	error("internal error: $file cannot be found");
+} #}}}
+
 sub readfile ($) { #{{{
 	my $file=shift;