Multiple changes

* Improved and cleaned up source import scripts
* Improved some reports and report overview
* New osmium version that uses a better hash function
* Some experimental magic to include Javascipt in an easier way (tested in keys.erb)

32 files changed, 443 insertions, 238 deletions

diff --git a/sources/db/osmium_tagstats b/sources/db/osmium_tagstats
index 618ee6d..ab5e21e 100755
--- a/sources/db/osmium_tagstats
+++ b/sources/db/osmium_tagstats
diff --git a/sources/db/post.sql b/sources/db/post.sql
index e182a28..e824c48 100644
--- a/sources/db/post.sql
+++ b/sources/db/post.sql
@@ -61,5 +61,5 @@ UPDATE keys SET prevalent_values=(
 
 ANALYZE;
 
-UPDATE meta SET update_end=datetime('now');
+UPDATE source SET update_end=datetime('now');
 
diff --git a/sources/db/pre.sql b/sources/db/pre.sql
index 998b20e..38ec051 100644
--- a/sources/db/pre.sql
+++ b/sources/db/pre.sql
@@ -6,24 +6,7 @@
 
 .bail ON
 
-DROP TABLE IF EXISTS meta;
-
-CREATE TABLE meta (
-    source_id    TEXT,
-    source_name  TEXT,
-    update_start TEXT,
-    update_end   TEXT,
-    data_until   TEXT
-);
-
-INSERT INTO meta (source_id, source_name, update_start) SELECT 'db', 'Database', datetime('now');
-
-DROP TABLE IF EXISTS stats;
-
-CREATE TABLE stats (
-    key   TEXT,
-    value INT64
-);
+INSERT INTO source (id, name, update_start) SELECT 'db', 'Database', datetime('now');
 
 DROP TABLE IF EXISTS keys;
 
diff --git a/sources/db/update.sh b/sources/db/update.sh
index 4ab2856..56a7652 100755
--- a/sources/db/update.sh
+++ b/sources/db/update.sh
@@ -9,32 +9,37 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start db: "; date
+echo "`$DATECMD` Start db..."
 
 DATABASE=$DIR/taginfo-db.db
 
 rm -f $DATABASE
 rm -f $DIR/count.db
 
-echo "Running pre.sql..."
+echo "`$DATECMD` Running init.sql..."
+sqlite3 $DATABASE <../init.sql
+
+echo "`$DATECMD` Running pre.sql..."
 sqlite3 $DATABASE <pre.sql
 
-echo -n "Running count... "; date
+echo "`$DATECMD` Running count... "
 HERE=`pwd`
 cd $DIR
 bzcat $DIR/planet.osm.bz2 | $HERE/osmium_tagstats -
 cd $HERE
 
-echo -n "Running update_characters... "; date
+echo "`$DATECMD` Running update_characters... "
 ./update_characters.pl $DIR
 
-echo -n "Running post.sql... "; date
+echo "`$DATECMD` Running post.sql... "
 perl -pe "s|__DIR__|$DIR|" post.sql | sqlite3 $DATABASE
 
-echo -n "Done db: "; date
+echo "`$DATECMD` Done db."
 
diff --git a/sources/init.sql b/sources/init.sql
new file mode 100644
index 0000000..e188915
--- /dev/null
+++ b/sources/init.sql
@@ -0,0 +1,29 @@
+--
+--  init.sql
+--
+--  Initialize source database
+--
+
+.bail ON
+
+--
+-- Contains metadata about this source.
+--
+DROP TABLE IF EXISTS source;
+CREATE TABLE source (
+    id           TEXT,
+    name         TEXT,
+    update_start TEXT,
+    update_end   TEXT,
+    data_until   TEXT
+);
+
+--
+-- Contains general statistical data for this source.
+--
+DROP TABLE IF EXISTS stats;
+CREATE TABLE stats (
+    key   TEXT,
+    value INT64
+);
+
diff --git a/sources/josm/post.sql b/sources/josm/post.sql
index 352b026..8dd6972 100644
--- a/sources/josm/post.sql
+++ b/sources/josm/post.sql
@@ -18,5 +18,5 @@ INSERT INTO stats (key, value) SELECT 'josm_tags_in_style_rules', count(distinct
 
 ANALYZE;
 
-UPDATE meta SET update_end=datetime('now');
+UPDATE source SET update_end=datetime('now');
 
diff --git a/sources/josm/pre.sql b/sources/josm/pre.sql
index 42a4213..5c2975d 100644
--- a/sources/josm/pre.sql
+++ b/sources/josm/pre.sql
@@ -6,24 +6,7 @@
 
 .bail ON
 
-DROP TABLE IF EXISTS meta;
-
-CREATE TABLE meta (
-    source_id    TEXT,
-    source_name  TEXT,
-    update_start TEXT,
-    update_end   TEXT,
-    data_until   TEXT
-);
-
-INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'josm', 'JOSM', datetime('now'), datetime('now');
-
-DROP TABLE IF EXISTS stats;
-
-CREATE TABLE stats (
-    key   TEXT,
-    value INT64
-);
+INSERT INTO source (id, name, update_start, data_until) SELECT 'josm', 'JOSM', datetime('now'), datetime('now');
 
 --
 --  josm_styles
diff --git a/sources/josm/update.sh b/sources/josm/update.sh
index 6eca722..abfb7e9 100755
--- a/sources/josm/update.sh
+++ b/sources/josm/update.sh
@@ -9,12 +9,14 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start josm: "; date
+echo "`$DATECMD` Start josm..."
 
 DATABASE=$DIR/taginfo-josm.db
 ELEMSTYLES=$DIR/elemstyles.xml
@@ -22,17 +24,20 @@ ELEMSTYLES=$DIR/elemstyles.xml
 rm -f $DATABASE
 rm -f $ELEMSTYLES
 
-echo "Getting styles..."
+echo "`$DATECMD` Getting styles..."
 wget -O $ELEMSTYLES http://josm.openstreetmap.de/svn/trunk/styles/standard/elemstyles.xml
 
-echo "Running pre.sql..."
+echo "`$DATECMD` Running init.sql..."
+sqlite3 $DATABASE <../init.sql
+
+echo "`$DATECMD` Running pre.sql..."
 sqlite3 $DATABASE <pre.sql
 
-echo "Running import..."
+echo "`$DATECMD` Running import..."
 ./import_josm.rb $DIR
 
-echo "Running post.sql..."
+echo "`$DATECMD` Running post.sql..."
 sqlite3 $DATABASE <post.sql
 
-echo -n "Done josm: "; date
+echo "`$DATECMD` Done josm."
 
diff --git a/sources/master/languages.sql b/sources/master/languages.sql
index 968e689..eb2b6d9 100644
--- a/sources/master/languages.sql
+++ b/sources/master/languages.sql
@@ -6,15 +6,17 @@
 
 .bail ON
 
+--
+--  Contains all the languages Taginfo knows about.
+--
 DROP TABLE IF EXISTS languages;
-
 CREATE TABLE languages (
     code           VARCHAR,
-    iso639_1       VARCHAR,
+    iso639_1       VARCHAR, -- official ISO 639-1 code (if available)
     english_name   VARCHAR,
     native_name    VARCHAR,
-    wiki_key_pages INTEGER,
-    wiki_tag_pages INTEGER
+    wiki_key_pages INTEGER, -- count of wiki pages with the title "code:Key:*" (or "Key:*" for code='en')
+    wiki_tag_pages INTEGER  -- count of wiki pages with the title "code:Tag:*" (or "Tag:*" for code='en')
 );
 
 INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('ar',      'ar', 'Arabic', 'العربية');
@@ -48,5 +50,3 @@ INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh',
 INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh-hans', NULL, 'Chinese', '中文');
 -- INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('', '', '', '');
 
-ANALYZE;
-
diff --git a/sources/master/master.sql b/sources/master/master.sql
index fc0ccef..c559426 100644
--- a/sources/master/master.sql
+++ b/sources/master/master.sql
@@ -1,8 +1,10 @@
+-- ============================================================================
 --
 --  Taginfo
 --
 --  master.sql
 --
+-- ============================================================================
 
 .bail ON
 
@@ -12,8 +14,29 @@ ATTACH DATABASE '__DIR__/josm/taginfo-josm.db'             AS josm;
 ATTACH DATABASE '__DIR__/potlatch/taginfo-potlatch.db'     AS potlatch; 
 ATTACH DATABASE '__DIR__/merkaartor/taginfo-merkaartor.db' AS merkaartor; 
 
-DROP TABLE IF EXISTS master_meta;
+-- ============================================================================
+
+--
+--  Collects information about all the sources.
+--
+DROP TABLE IF EXISTS sources;
+CREATE TABLE sources (
+    no           INTEGER,
+    visible      INTEGER,
+    id           TEXT,
+    name         TEXT,
+    update_start TEXT,
+    update_end   TEXT,
+    data_until   TEXT
+);
+
+INSERT INTO sources SELECT 1, 1, * FROM db.source
+              UNION SELECT 2, 1, * FROM wiki.source
+              UNION SELECT 3, 1, * FROM josm.source
+              UNION SELECT 4, 0, * FROM potlatch.source
+              UNION SELECT 5, 0, * FROM merkaartor.source;
 
+DROP TABLE IF EXISTS master_meta;
 CREATE TABLE master_meta (
     source_id    TEXT,
     source_name  TEXT,
@@ -22,14 +45,14 @@ CREATE TABLE master_meta (
     data_until   TEXT
 );
 
-INSERT INTO master_meta SELECT * FROM db.meta
-                  UNION SELECT * FROM wiki.meta
-                  UNION SELECT * FROM josm.meta;
+-- for backwards compatibility
+INSERT INTO master_meta SELECT * FROM db.source
+                  UNION SELECT * FROM wiki.source
+                  UNION SELECT * FROM josm.source;
 -- XXX                 UNION SELECT * FROM potlatch.meta
 -- XXX                 UNION SELECT * FROM merkaartor.meta;
 
 DROP TABLE IF EXISTS master_stats;
-
 CREATE TABLE master_stats (
     key   TEXT,
     value INT64
@@ -41,6 +64,8 @@ INSERT INTO master_stats SELECT * FROM db.stats
                    UNION SELECT * FROM potlatch.stats
                    UNION SELECT * FROM merkaartor.stats;
 
+-- ============================================================================
+
 INSERT INTO db.keys (key) SELECT DISTINCT key FROM wiki.wikipages        WHERE key NOT IN (SELECT key FROM db.keys);
 INSERT INTO db.keys (key) SELECT DISTINCT k   FROM josm.josm_style_rules WHERE k   NOT IN (SELECT key FROM db.keys);
 -- potlatch XXX
@@ -55,9 +80,9 @@ UPDATE db.keys SET in_merkaartor=1 WHERE key IN (SELECT key FROM merkaartor.keys
 -- too slow, so we drop it for now
 -- INSERT INTO db.tags (key, value) SELECT DISTINCT key, value FROM wiki.wikipages WHERE key || '=XX=' || value NOT IN (SELECT key || '=XX=' || value FROM db.tags);
 
+-- ============================================================================
 
 DROP TABLE IF EXISTS popular_keys;
-
 CREATE TABLE popular_keys (
     key           VARCHAR,
     count         INTEGER,
@@ -87,8 +112,9 @@ UPDATE popular_keys SET in_wiki=1    WHERE key IN (SELECT distinct key FROM wiki
 UPDATE popular_keys SET in_wiki_en=1 WHERE key IN (SELECT distinct key FROM wiki.wikipages WHERE lang='en');
 UPDATE popular_keys SET in_josm=1    WHERE key IN (SELECT distinct k   FROM josm.josm_style_rules);
 
-DROP TABLE IF EXISTS popular_metadata;
+-- ============================================================================
 
+DROP TABLE IF EXISTS popular_metadata;
 CREATE TABLE popular_metadata (
     keys        INTEGER,
     count_min   INTEGER,
@@ -111,9 +137,13 @@ UPDATE popular_keys SET scale_name  = 0 WHERE key LIKE '%:%';
 
 UPDATE popular_keys SET scale1 = 10 * scale_count + 8 * scale_users + 2 * scale_wiki + 1 * scale_josm + 2 * scale_name;
 
+-- ============================================================================
+
 INSERT INTO languages (code) SELECT distinct(lang) FROM wiki.wikipages WHERE lang NOT IN (SELECT code FROM languages);
 UPDATE languages SET wiki_key_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NULL);
 UPDATE languages SET wiki_tag_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NOT NULL);
 
+-- ============================================================================
+
 ANALYZE;
 
diff --git a/sources/master/update.sh b/sources/master/update.sh
index b8d63b2..f1598d9 100755
--- a/sources/master/update.sh
+++ b/sources/master/update.sh
@@ -9,12 +9,14 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start master: "; date
+echo "`$DATECMD` Start master..."
 
 DATABASE=$DIR/taginfo-master.db
 
@@ -23,5 +25,5 @@ rm -f $DATABASE
 sqlite3 $DATABASE <languages.sql
 perl -pe "s|__DIR__|$DIR|" master.sql | sqlite3 $DATABASE
 
-echo -n "Done master: "; date
+echo "`$DATECMD` Done master."
 
diff --git a/sources/merkaartor/post.sql b/sources/merkaartor/post.sql
index 9e7fea9..06bcef0 100644
--- a/sources/merkaartor/post.sql
+++ b/sources/merkaartor/post.sql
@@ -6,3 +6,7 @@
 
 .bail ON
 
+ANALYZE;
+
+UPDATE source SET update_end=datetime('now');
+
diff --git a/sources/merkaartor/pre.sql b/sources/merkaartor/pre.sql
index 60ba309..76d8012 100644
--- a/sources/merkaartor/pre.sql
+++ b/sources/merkaartor/pre.sql
@@ -6,25 +6,7 @@
 
 .bail ON
 
-DROP TABLE IF EXISTS meta;
-
-CREATE TABLE meta (
-    source_id    TEXT,
-    source_name  TEXT,
-    update_start TEXT,
-    update_end   TEXT,
-    data_until   TEXT
-);
-
-INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now');
-
-DROP TABLE IF EXISTS stats;
-
-CREATE TABLE stats (
-    key   TEXT,
-    value INT64
-);
-
+INSERT INTO source (id, name, update_start, data_until) SELECT 'merkaartor', 'Merkaartor', datetime('now'), datetime('now');
 
 --
 --  templates
diff --git a/sources/merkaartor/update.sh b/sources/merkaartor/update.sh
index ae97d45..a112b27 100755
--- a/sources/merkaartor/update.sh
+++ b/sources/merkaartor/update.sh
@@ -9,18 +9,20 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start merkaartor: "; date
+echo "`$DATECMD` Start merkaartor..."
 
 DATABASE=$DIR/taginfo-merkaartor.db
 
 rm -f $DATABASE
 
-echo "Getting resources..."
+echo "`$DATECMD` Updating resources..."
 if [ -d $DIR/git-source ]; then
     cd $DIR/git-source
     git pull
@@ -29,14 +31,17 @@ else
     git clone http://git.gitorious.org/merkaartor/main.git $DIR/git-source
 fi
 
-echo "Running pre.sql..."
+echo "`$DATECMD` Running init.sql..."
+sqlite3 $DATABASE <../init.sql
+
+echo "`$DATECMD` Running pre.sql..."
 sqlite3 $DATABASE <pre.sql
 
-echo "Running import..."
+echo "`$DATECMD` Running import..."
 ./import_merkaartor.rb $DIR
 
-echo "Running post.sql..."
+echo "`$DATECMD` Running post.sql..."
 sqlite3 $DATABASE <post.sql
 
-echo -n "Done merkaartor: "; date
+echo "`$DATECMD` Done merkaartor."
 
diff --git a/sources/potlatch/post.sql b/sources/potlatch/post.sql
index 3446d08..1254b89 100644
--- a/sources/potlatch/post.sql
+++ b/sources/potlatch/post.sql
@@ -11,3 +11,5 @@ UPDATE features SET category_name = (SELECT name FROM categories WHERE id=catego
 
 ANALYZE;
 
+UPDATE source SET update_end=datetime('now');
+
diff --git a/sources/potlatch/pre.sql b/sources/potlatch/pre.sql
index 9f79445..35d8c9d 100644
--- a/sources/potlatch/pre.sql
+++ b/sources/potlatch/pre.sql
@@ -6,24 +6,7 @@
 
 .bail ON
 
-DROP TABLE IF EXISTS meta;
-
-CREATE TABLE meta (
-    source_id    TEXT,
-    source_name  TEXT,
-    update_start TEXT,
-    update_end   TEXT,
-    data_until   TEXT
-);
-
-INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now');
-
-DROP TABLE IF EXISTS stats;
-
-CREATE TABLE stats (
-    key   TEXT,
-    value INT64
-);
+INSERT INTO source (id, name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now');
 
 --
 --  categories
diff --git a/sources/potlatch/update.sh b/sources/potlatch/update.sh
index 8ba9956..748ff4a 100755
--- a/sources/potlatch/update.sh
+++ b/sources/potlatch/update.sh
@@ -9,32 +9,37 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start potlatch: "; date
+echo "`$DATECMD` Start potlatch..."
 
 DATABASE=$DIR/taginfo-potlatch.db
 
 rm -f $DATABASE
 
-echo "Getting resources..."
+echo "`$DATECMD` Updating resources..."
 if [ -d $DIR/resources ]; then
     svn update $DIR/resources
 else
     svn checkout http://svn.openstreetmap.org/applications/editors/potlatch2/resources $DIR/resources
 fi
 
-echo "Running pre.sql..."
+echo "`$DATECMD` Running init.sql..."
+sqlite3 $DATABASE <../init.sql
+
+echo "`$DATECMD` Running pre.sql..."
 sqlite3 $DATABASE <pre.sql
 
-echo "Running import..."
+echo "`$DATECMD` Running import..."
 ./import_potlatch.rb $DIR
 
-echo "Running post.sql..."
+echo "`$DATECMD` Running post.sql..."
 sqlite3 $DATABASE <post.sql
 
-echo -n "Done potlatch: "; date
+echo "`$DATECMD` Done potlatch."
 
diff --git a/sources/update_all.sh b/sources/update_all.sh
index f7e4459..39e58ce 100755
--- a/sources/update_all.sh
+++ b/sources/update_all.sh
@@ -9,6 +9,8 @@ SOURCES="josm potlatch merkaartor wiki db"
 
 set -e
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 DIR=$1
 
 if [ "x" = "x$DIR" ]; then
@@ -16,9 +18,11 @@ if [ "x" = "x$DIR" ]; then
     exit 1
 fi
 
-exec >$DIR/update_all.log 2>&1
+LOGFILE=`date +%Y%m%dT%H%M`
+mkdir -p $DIR/log
+exec >$DIR/log/$LOGFILE.log 2>&1
 
-echo -n "Start: "; date
+echo "`$DATECMD` Start update_all..."
 
 mkdir -p $DIR/download
 
@@ -40,7 +44,7 @@ cd ..
 
 for source in $SOURCES; do
     echo "====================================="
-    echo "Running bzip2..."
+    echo "Running bzip2 on $source..."
     bzip2 -9 -c $DIR/$source/taginfo-$source.db >$DIR/download/taginfo-$source.db.bz2
     echo "Done."
 done
@@ -49,5 +53,6 @@ echo "Running bzip2..."
 bzip2 -9 -c $DIR/taginfo-master.db >$DIR/download/taginfo-master.db.bz2
 echo "Done."
 
-echo -n "Done: "; date
+echo "====================================="
+echo "`$DATECMD` Done update_all."
 
diff --git a/sources/wiki/post.sql b/sources/wiki/post.sql
index eaf9ec0..773a04d 100644
--- a/sources/wiki/post.sql
+++ b/sources/wiki/post.sql
@@ -36,5 +36,5 @@ INSERT INTO stats (key, value) SELECT 'wiki_languages', count(*) FROM wiki_langu
 
 ANALYZE;
 
-UPDATE meta SET update_end=datetime('now');
+UPDATE source SET update_end=datetime('now');
 
diff --git a/sources/wiki/pre.sql b/sources/wiki/pre.sql
index 0e8200e..56e9538 100644
--- a/sources/wiki/pre.sql
+++ b/sources/wiki/pre.sql
@@ -6,24 +6,7 @@
 
 .bail ON
 
-DROP TABLE IF EXISTS meta;
-
-CREATE TABLE meta (
-    source_id    TEXT,
-    source_name  TEXT,
-    update_start TEXT,
-    update_end   TEXT,
-    data_until   TEXT
-);
-
-INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'wiki', 'Wiki', datetime('now'), datetime('now');
-
-DROP TABLE IF EXISTS stats;
-
-CREATE TABLE stats (
-    key   TEXT,
-    value INT64
-);
+INSERT INTO source (id, name, update_start, data_until) SELECT 'wiki', 'Wiki', datetime('now'), datetime('now');
 
 DROP TABLE IF EXISTS wikipages;
 
diff --git a/sources/wiki/update.sh b/sources/wiki/update.sh
index b6f63a6..7d7dc6b 100755
--- a/sources/wiki/update.sh
+++ b/sources/wiki/update.sh
@@ -9,12 +9,14 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start wiki: "; date
+echo "`$DATECMD` Start wiki..."
 
 DATABASE=$DIR/taginfo-wiki.db
 LOGFILE=$DIR/get_wiki_data.log
@@ -24,17 +26,20 @@ rm -f $DIR/tagpages.list
 rm -f $LOGFILE
 rm -f $DATABASE
 
-echo "Running pre.sql..."
+echo "`$DATECMD` Running init.sql..."
+sqlite3 $DATABASE <../init.sql
+
+echo "`$DATECMD` Running pre.sql..."
 sqlite3 $DATABASE <pre.sql
 
-echo "Getting page list..."
+echo "`$DATECMD` Getting page list..."
 ./get_page_list.rb $DIR
 
-echo "Getting wiki data..."
+echo "`$DATECMD` Getting wiki data..."
 ./get_wiki_data.rb $DIR >$LOGFILE
 
-echo "Running post.sql..."
+echo "`$DATECMD` Running post.sql..."
 sqlite3 $DATABASE <post.sql
 
-echo -n "Done wiki: "; date
+echo "`$DATECMD` Done wiki."
 
diff --git a/web/lib/javascript.rb b/web/lib/javascript.rb
new file mode 100644
index 0000000..7f43d44
--- /dev/null
+++ b/web/lib/javascript.rb
@@ -0,0 +1,88 @@
+# web/lib/javascript.rb
+
+def javascript(url=nil, &block)
+    @javascript ||= Array.new
+    @javascript << Javascript.new(url, &block)
+end
+
+def javascript_tags
+    @javascript.map{ |js| js.to_html }
+end
+
+class Javascript
+
+    def initialize(file)
+        if file.nil?
+            c = ''
+            r = yield c
+            @content = c + r
+        else
+            @file = file
+        end
+    end
+
+    def to_html
+        if @file.nil?
+            %Q{        <script type="text/javascript">//<![CDATA[\n#{ @content }//]]></script>\n}
+        else
+            %Q{        <script type="text/javascript" src="/js/#{ @file }.js"></script>\n}
+        end
+    end
+
+end
+
+class JQuery
+
+    # "include" the convenience methods from R18n::Helpers.
+    # Uses extend instead of include, because we want this
+    # to work not with instances of JQuery but they should
+    # show up as JQuery class methods. 
+    extend R18n::Helpers
+
+    def self.ready
+        "jQuery(document).ready(function() {\n" +
+            yield +
+        "});\n"
+    end
+
+    def self.flexigrid(id, options)
+        defaults = {
+            :method        => 'GET',
+            :dataType      => 'json',
+            :pagetext      => t.flexigrid.pagetext,
+            :pagestat      => t.flexigrid.pagestat,
+            :outof         => t.flexigrid.outof,
+            :findtext      => t.flexigrid.findtext,
+            :procmsg       => t.flexigrid.procmsg,
+            :nomsg         => t.flexigrid.nomsg,
+            :errormsg      => t.flexigrid.errormsg,
+            :showToggleBtn => false,
+            :usepager      => true,
+            :useRp         => true,
+            :rp            => 15,
+            :rpOptions     => [10,15,20,25,50,100],
+        }
+        "jQuery('##{id}').flexigrid(" + defaults.merge(options).to_json + ");\n"
+    end
+
+end
+
+class JS
+
+    #
+    #  Careful, deep magic!
+    #
+    #  We redefine the to_json method of the String argument to return
+    #  the raw string. This way we can do JS.raw("foo").to_json and get "foo".
+    #
+    def self.raw(code)
+       code.instance_eval do
+          def to_json
+              to_s
+          end
+       end
+       code
+    end
+
+end
+
diff --git a/web/lib/reports.rb b/web/lib/reports.rb
index 99d500a..0895359 100644
--- a/web/lib/reports.rb
+++ b/web/lib/reports.rb
@@ -11,15 +11,19 @@ class Report
         end
     end
 
-    def initialize(title, sources)
+    def initialize(title, *sources)
         @@reports << self
         @title = title
         @sources = Hash.new
-        sources.each do |s|
-            @sources[s] = 1
+        sources.each do |id|
+            @sources[id] = 1
         end
     end
 
+    def uses_source?(id)
+        sources.has_key? id
+    end
+
     def name
         @title.gsub(/ /, '_').downcase
     end
@@ -30,9 +34,10 @@ class Report
 
 end
 
-Report.new 'Characters in Keys', %w(db)
-Report.new 'Frequently Used Keys Without Wiki Page', %w(db wiki)
-Report.new 'Key Lengths', %w(db)
-Report.new 'Language Comparison Table for Keys in the Wiki', %w(wiki)
-Report.new 'Languages', %w(wiki)
-Report.new 'Wiki Pages About Non-Existing Keys', %w(db wiki)
+Report.new 'Characters in Keys', :db
+Report.new 'Frequently Used Keys Without Wiki Page', :db, :wiki
+Report.new 'Key Lengths', :db
+Report.new 'Language Comparison Table for Keys in the Wiki', :wiki
+Report.new 'Languages', :wiki
+Report.new 'Wiki Pages About Non-Existing Keys', :db, :wiki
+
diff --git a/web/lib/sources.rb b/web/lib/sources.rb
new file mode 100644
index 0000000..918dd4d
--- /dev/null
+++ b/web/lib/sources.rb
@@ -0,0 +1,57 @@
+# web/lib/sources.rb
+class Source
+
+    @@sources = Array.new
+
+    attr_reader :id, :name, :data_until, :update_start, :update_end
+
+    # Enumerate all available sources
+    def self.each
+        @@sources.each do |source|
+            yield source
+        end
+    end
+
+    # The number of available sources
+    def self.size
+        @@sources.size
+    end
+
+    # Create new source
+    #  id - Symbol with id for this source
+    #  name - Name of this source
+    def initialize(id, name, data_until, update_start, update_end)
+        @id           = id.to_sym
+        @name         = name
+        @data_until   = data_until
+        @update_start = update_start
+        @update_end   = update_end
+        @@sources << self
+    end
+
+    # The URL where this source is described
+    def url
+        "/sources/#{ @id }"
+    end
+
+    # The img URL of this source
+    def imgurl(size=16)
+        "/img/sources/#{ @id }.#{ size }.png"
+    end
+
+    # Returns img tag for this source
+    def img(size=16)
+        %Q{<img src="#{ imgurl(size) }" alt="#{ name }" title="#{ name }" width="#{ size }" height="#{ size }"/>}
+    end
+
+    # Returns a link to this source
+    def link_img
+        %Q{<a href="#{ url }">#{ img }</a>}
+    end
+
+    # Returns a link to this source
+    def link_name
+        %Q{<a href="#{ url }">#{ name }</a>}
+    end
+
+end
diff --git a/web/public/css/taginfo.css b/web/public/css/taginfo.css
index c125799..b5d69e8 100644
--- a/web/public/css/taginfo.css
+++ b/web/public/css/taginfo.css
@@ -28,8 +28,8 @@ p {
     margin-top: 2px;
 }
 
-dd {
-    padding-bottom: 12px;
+p.text {
+    max-width: 60em;
 }
 
 iframe {
@@ -311,6 +311,7 @@ table.list {
 
 table.list th {
     padding: 4px;
+    font-weight: normal;
 }
 
 table.list td {
@@ -325,6 +326,44 @@ table.list td.border {
 
 /* ========== */
 
+table.reports {
+    border-collapse: collapse;
+}
+
+table.reports th.source {
+    background-color: #f0f0f0;
+    font-size: 80%;
+    padding-bottom: 4px;
+    -moz-border-radius: 4px 4px 0 0;
+    -khtml-border-radius: 4px 4px 0 0;
+    -webkit-border-radius: 4px 4px 0 0;
+    -chrome-border-radius: 4px 4px 0 0;
+    -o-border-radius: 4px 4px 0 0;
+}
+
+table.reports td {
+    padding: 2px 4px;
+}
+
+table.reports td.source {
+    background-color: #f0f0f0;
+    text-align: center;
+    padding: 2px 0 0 0;
+    width: 24px;
+}
+
+table.reports th.last {
+    background-color: #f0f0f0;
+    height: 8px;
+    -moz-border-radius: 0 0 4px 4px;
+    -khtml-border-radius: 0 0 4px 4px;
+    -webkit-border-radius: 0 0 4px 4px;
+    -chrome-border-radius: 0 0 4px 4px;
+    -o-border-radius: 0 0 4px 4px;
+}
+
+/* ========== */
+
 div#canvas-values {
     position: absolute;
     border: 1px solid #cccccc;
diff --git a/web/public/img/sources/wiki/wiki-e.png b/web/public/img/sources/wiki/wiki-e.png
index 63e468e..1ec9fb0 100644
--- a/web/public/img/sources/wiki/wiki-e.png
+++ b/web/public/img/sources/wiki/wiki-e.png
diff --git a/web/taginfo.rb b/web/taginfo.rb
index 8bc5233..a3de09b 100755
--- a/web/taginfo.rb
+++ b/web/taginfo.rb
@@ -35,8 +35,10 @@ require 'sinatra/base'
 require 'sinatra/r18n'
 
 require 'lib/utils.rb'
+require 'lib/javascript.rb'
 require 'lib/language.rb'
 require 'lib/sql.rb'
+require 'lib/sources.rb'
 require 'lib/reports.rb'
 
 #------------------------------------------------------------------------------
@@ -45,6 +47,15 @@ TAGCLOUD_NUMBER_OF_TAGS = 200
 
 #------------------------------------------------------------------------------
 
+db = SQL::Database.new('../../data')
+
+# XXX update this to use new sources table
+db.select('SELECT * FROM master_meta ORDER BY source_name').execute().each do |source|
+    Source.new source['source_id'], source['source_name'], source['data_until'], source['update_start'], source['update_end']
+end
+
+db.close
+
 class Taginfo < Sinatra::Base
 
     register Sinatra::R18n
@@ -84,6 +95,12 @@ class Taginfo < Sinatra::Base
 #        end
         params[:locale] = 'en'
 
+        javascript 'jquery-1.3.2.min'
+        javascript 'jquery-ui-1.7.2.all.min'
+        javascript 'flexigrid-minified'
+        javascript 'taginfo'
+        javascript 'protovis-r3.2'
+
         @db = SQL::Database.new('../../data')
 
         @data_until = @db.select("SELECT min(data_until) FROM master_meta").get_first_value().sub(/:..$/, '')
diff --git a/web/views/keys.erb b/web/views/keys.erb
index ced0a96..e398fa0 100644
--- a/web/views/keys.erb
+++ b/web/views/keys.erb
@@ -3,63 +3,48 @@
 <table id="keys">
 </table>
 
-<script type="text/javascript">
-    jQuery(document).ready(function() {
-        jQuery('#keys').flexigrid({
-            url: '/api/2/db/keys',
-            method: 'GET',
-            dataType: 'json',
-            colModel: [
-                { display: 'Key', name: 'key', width: 180, sortable: true, align: 'left' },
-                { display: '<span title="Number of objects with this key"><img src="/img/types/all.16.png" alt=""/> Total</span>',           name: 'count_all',        width: 250, sortable: true, align: 'center' },
-                { display: '<span title="Number of nodes with this key"><img src="/img/types/node.16.png" alt=""/> Nodes</span>',            name: 'count_nodes',      width: 250, sortable: true, align: 'center' },
-                { display: '<span title="Number of ways with this key"><img src="/img/types/way.16.png" alt=""/> Ways</span>',               name: 'count_ways',       width: 250, sortable: true, align: 'center' },
-                { display: '<span title="Number of relations with this key"><img src="/img/types/relation.16.png" alt=""/> Relation</span>', name: 'count_relations',  width: 250, sortable: true, align: 'center' },
-                { display: 'Users', name: 'users_all', width: 44, sortable: true, align: 'right' },
-                { display: '<img src="/img/sources/wiki.16.png" alt="Wiki" title="Wiki"/>', name: 'in_wiki', width: 20, sortable: true, align: 'center' },
-                { display: '<img src="/img/sources/josm.16.png" alt="JOSM" title="JOSM"/>', name: 'in_josm', width: 20, sortable: true, align: 'center' },
-             //   { display: '<img src="/img/sources/potlatch.16.png" alt="Potlatch 2" title="Potlatch 2"/>', name: 'in_potlatch', width: 20, sortable: true, align: 'center' },
-             //   { display: '<img src="/img/sources/merkaartor.16.png" alt="Merkaartor" title="Merkaartor"/>', name: 'in_merkaartor', width: 20, sortable: true, align: 'center' },
-                { display: '<span title="Number of different values for this key">Values</span>', name: 'values_all', width: 70, sortable: true, align: 'right' },
-                { display: 'Prevalent Values', name: 'prevalent_values', width: 500, sortable: true, align: 'left' }
+<% javascript do
+    JQuery.ready do
+        JQuery.flexigrid(:keys,
+            :url => '/api/2/db/keys',
+            :colModel => [
+                { :display => 'Key', :name => 'key', :width => 180, :sortable => true, :align => 'left' },
+                { :display => '<span title="Number of objects with this key"><img src="/img/types/all.16.png" alt=""/> Total</span>',           :name => 'count_all',        :width => 250, :sortable => true, :align => 'center' },
+                { :display => '<span title="Number of nodes with this key"><img src="/img/types/node.16.png" alt=""/> Nodes</span>',            :name => 'count_nodes',      :width => 250, :sortable => true, :align => 'center' },
+                { :display => '<span title="Number of ways with this key"><img src="/img/types/way.16.png" alt=""/> Ways</span>',               :name => 'count_ways',       :width => 250, :sortable => true, :align => 'center' },
+                { :display => '<span title="Number of relations with this key"><img src="/img/types/relation.16.png" alt=""/> Relation</span>', :name => 'count_relations',  :width => 250, :sortable => true, :align => 'center' },
+                { :display => 'Users', :name => 'users_all', :width => 44, :sortable => true, :align => 'right' },
+                { :display => '<img src="/img/sources/wiki.16.png" alt="Wiki" title="Wiki"/>', :name => 'in_wiki', :width => 20, :sortable => true, :align => 'center' },
+                { :display => '<img src="/img/sources/josm.16.png" alt="JOSM" title="JOSM"/>', :name => 'in_josm', :width => 20, :sortable => true, :align => 'center' },
+                { :display => '<span title="Number of different values for this key">Values</span>', :name => 'values_all', :width => 70, :sortable => true, :align => 'right' },
+                { :display => 'Prevalent Values', :name => 'prevalent_values', :width => 500, :sortable => true, :align => 'left' }
             ],
-            searchitems: [
-                { display: 'Key', name: 'key' }
+            :searchitems => [
+                { :display => 'Key', :name => 'key' }
             ],
-            sortname: 'count_all',
-            sortorder: 'desc',
-            pagetext: '<%= t.flexigrid.pagetext %>',
-            pagestat: '<%= t.flexigrid.pagestat %>',
-            outof:    '<%= t.flexigrid.outof    %>',
-            findtext: '<%= t.flexigrid.findtext %>',
-            procmsg:  '<%= t.flexigrid.procmsg  %>',
-            nomsg:    '<%= t.flexigrid.nomsg    %>',
-            errormsg: '<%= t.flexigrid.errormsg %>',
-            showToggleBtn: false,
-            usepager: true,
-            useRp: true,
-            rp: 15,
-            rpOptions: [10,15,20,25,50,100],
-            height: 420,
-            preProcess: function(data) {
-                data.rows = jQuery.map(data.data, function(row, i) {
-                    return { 'cell': [
-                        link_to_key(row.key),
-                        print_value_with_percent(row.count_all,       row.count_all_fraction),
-                        print_value_with_percent(row.count_nodes,     row.count_nodes_fraction),
-                        print_value_with_percent(row.count_ways,      row.count_ways_fraction),
-                        print_value_with_percent(row.count_relations, row.count_relations_fraction),
-                        print_with_ts(row.users_all),
-                        row.in_wiki       ? '&#x2714;' : '-',
-                        row.in_josm       ? '&#x2714;' : '-',
-                 //       row.in_potlatch   ? '&#x2714;' : '-',
-                 //       row.in_merkaartor ? '&#x2714;' : '-',
-                        print_with_ts(row.values_all),
-                        print_tag_list(row.key, row.prevalent_values)
-                    ] };
-                });
-                return data;
-            }
-        });
+            :sortname => 'count_all',
+            :sortorder => 'desc',
+            :height => 420,
+            :preProcess => JS.raw(%q{
+function(data) {
+    data.rows = jQuery.map(data.data, function(row, i) {
+        return { 'cell': [
+            link_to_key(row.key),
+            print_value_with_percent(row.count_all,       row.count_all_fraction),
+            print_value_with_percent(row.count_nodes,     row.count_nodes_fraction),
+            print_value_with_percent(row.count_ways,      row.count_ways_fraction),
+            print_value_with_percent(row.count_relations, row.count_relations_fraction),
+            print_with_ts(row.users_all),
+            row.in_wiki ? '&#x2714;' : '-',
+            row.in_josm ? '&#x2714;' : '-',
+            print_with_ts(row.values_all),
+            print_tag_list(row.key, row.prevalent_values)
+        ] };
     });
-</script>
+    return data;
+}
+            })
+        )
+    end
+end
+%>
diff --git a/web/views/layout.erb b/web/views/layout.erb
index e2fc109..ebbe352 100644
--- a/web/views/layout.erb
+++ b/web/views/layout.erb
@@ -5,13 +5,9 @@
         <meta http-equiv="content-type" content="text/html; charset=utf-8" />
         <meta name="content-language" content="<%= r18n.locale.code %>" />
         <link rel="stylesheet" type="text/css" href="/css/taginfo.css" />
-        <link rel="Stylesheet" type="text/css" href="/css/smoothness/jquery-ui-1.7.3.custom.css" />
+        <link rel="stylesheet" type="text/css" href="/css/smoothness/jquery-ui-1.7.3.custom.css" />
         <link rel="stylesheet" type="text/css" href="/css/flexigrid/flexigrid.css" />
-        <script type="text/javascript" src="/js/jquery-1.3.2.min.js"></script>
-        <script type="text/javascript" src="/js/jquery-ui-1.7.2.all.min.js"></script>
-        <script type="text/javascript" src="/js/flexigrid-minified.js"></script>
-        <script type="text/javascript" src="/js/taginfo.js"></script>
-        <script type="text/javascript" src="/js/protovis-r3.2.js"></script>
+<%= javascript_tags %>
     </head>
     <body>
         <div id="top">
diff --git a/web/views/reports/characters_in_keys.erb b/web/views/reports/characters_in_keys.erb
index b54a0a4..8f0b578 100644
--- a/web/views/reports/characters_in_keys.erb
+++ b/web/views/reports/characters_in_keys.erb
@@ -51,6 +51,12 @@ case latin letters (a-z) and the underscore (_) are used. Sometimes the colon
                 <td><%= sprintf '%.2f', @db.stats('characters_in_keys_rest') * 100.0 / @db.stats('num_keys') %>%</td>
                 <td>Everything else.</td>
             </tr>
+            <tr>
+                <td style="border-top: 1px solid #000000;"></td>
+                <td style="border-top: 1px solid #000000;"><%= @db.stats('num_keys') %></td>
+                <td style="border-top: 1px solid #000000;">100%</td>
+                <td style="border-top: 1px solid #000000;">Total</td>
+            </tr>
         </table>
     </div>
     <div id="tabs-whitespace">
@@ -90,7 +96,7 @@ case latin letters (a-z) and the underscore (_) are used. Sometimes the colon
                 { display: 'Characters in Key', name: 'characters', width: 810, sortable: true, align: 'left' }
             ],
             width: 990,
-            height: 170
+            height: 200
         });
         jQuery('#keys-whitespace').flexigrid({
             url: '/api/2/db/keys?filters=characters_space',
diff --git a/web/views/reports/index.erb b/web/views/reports/index.erb
index b9642e8..d77a4d2 100644
--- a/web/views/reports/index.erb
+++ b/web/views/reports/index.erb
@@ -1,19 +1,21 @@
 <h1>Reports</h1>
 
-<p>Reports show the data from different perspectives. They often bring
-together the data from several sources in interesting ways.</p>
+<p class="text">Reports show the data from different perspectives.
+They often bring together data from several sources in interesting ways. Some
+of the reports can help with finding certain kinds of errors, such as popular
+keys without wiki pages.</p>
 
-<table>
-    <% Report.each do |report| %>
-        <tr>
-            <% %w(db wiki).each do |source| %>
-                <td>
-                    <% if report.sources[source] %>
-                        <img src="/img/sources/<%= source %>.16.png" alt="<%= source.capitalize %>" title="<%= source.capitalize %>" width="16" height="16"/>
-                    <% end %>
-                </td>
-            <% end %>
-            <td style="padding: 2px;"><a href="<%= report.url %>"><%= report.title %></a></td>
-        </tr>
-    <% end %>
+<table class="reports">
+    <tr><th class="source" colspan="<%= Source.size %>"/>Data<br/>Sources</th><th></th></tr>
+<% Report.each do |report| %>
+    <tr>
+<% Source.each do |source| %>
+        <td class="source"><%= source.link_img if report.uses_source? source.id %></td>
+<% end %>
+        <td><a href="<%= report.url %>"><%= report.title %></a></td>
+    </tr>
+<% end %>
+    <tr><th class="last" colspan="<%= Source.size %>"/></th><th></th></tr>
 </table>
+
+
diff --git a/web/views/sources/index.erb b/web/views/sources/index.erb
index ba7fdb8..ee40a60 100644
--- a/web/views/sources/index.erb
+++ b/web/views/sources/index.erb
@@ -10,13 +10,12 @@ have to go back to the source.</p>
     <tr>
         <th></th><th>Source</th><th>Data until*</th><th>Last update run</th><th></th>
     </tr>
-    <% @sources.each do |source| %>
+    <% Source.each do |source| %>
     <tr>
         <td class="border"></td>
-        <td><a href="/sources/<%= source['source_id'] %>"><img src="/img/sources/<%= source['source_id'] %>.16.png" alt="" width="16" height="16"/></a>
-            <a href="/sources/<%= source['source_id'] %>"><%= source['source_name'] %></a></td>
-        <td><%= source['data_until'] %></td>
-        <td><%= source['update_start'] %> to <%= source['update_end'] %></td>
+        <td><%= source.link_img + ' ' + source.link_name %>
+        <td><%= source.data_until %></td>
+        <td><%= source.update_start %> to <%= source.update_end %></td>
         <td class="border"></td>
     </tr>
     <% end %>