Multiple changes

* Improved and cleaned up source import scripts
* Improved some reports and report overview
* New osmium version that uses a better hash function
* Some experimental magic to include Javascipt in an easier way (tested in keys.erb)

3 files changed, 47 insertions, 15 deletions

diff --git a/sources/master/languages.sql b/sources/master/languages.sql
index 968e689..eb2b6d9 100644
--- a/sources/master/languages.sql
+++ b/sources/master/languages.sql
@@ -6,15 +6,17 @@
 
 .bail ON
 
+--
+--  Contains all the languages Taginfo knows about.
+--
 DROP TABLE IF EXISTS languages;
-
 CREATE TABLE languages (
     code           VARCHAR,
-    iso639_1       VARCHAR,
+    iso639_1       VARCHAR, -- official ISO 639-1 code (if available)
     english_name   VARCHAR,
     native_name    VARCHAR,
-    wiki_key_pages INTEGER,
-    wiki_tag_pages INTEGER
+    wiki_key_pages INTEGER, -- count of wiki pages with the title "code:Key:*" (or "Key:*" for code='en')
+    wiki_tag_pages INTEGER  -- count of wiki pages with the title "code:Tag:*" (or "Tag:*" for code='en')
 );
 
 INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('ar',      'ar', 'Arabic', 'العربية');
@@ -48,5 +50,3 @@ INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh',
 INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh-hans', NULL, 'Chinese', '中文');
 -- INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('', '', '', '');
 
-ANALYZE;
-
diff --git a/sources/master/master.sql b/sources/master/master.sql
index fc0ccef..c559426 100644
--- a/sources/master/master.sql
+++ b/sources/master/master.sql
@@ -1,8 +1,10 @@
+-- ============================================================================
 --
 --  Taginfo
 --
 --  master.sql
 --
+-- ============================================================================
 
 .bail ON
 
@@ -12,8 +14,29 @@ ATTACH DATABASE '__DIR__/josm/taginfo-josm.db'             AS josm;
 ATTACH DATABASE '__DIR__/potlatch/taginfo-potlatch.db'     AS potlatch; 
 ATTACH DATABASE '__DIR__/merkaartor/taginfo-merkaartor.db' AS merkaartor; 
 
-DROP TABLE IF EXISTS master_meta;
+-- ============================================================================
+
+--
+--  Collects information about all the sources.
+--
+DROP TABLE IF EXISTS sources;
+CREATE TABLE sources (
+    no           INTEGER,
+    visible      INTEGER,
+    id           TEXT,
+    name         TEXT,
+    update_start TEXT,
+    update_end   TEXT,
+    data_until   TEXT
+);
+
+INSERT INTO sources SELECT 1, 1, * FROM db.source
+              UNION SELECT 2, 1, * FROM wiki.source
+              UNION SELECT 3, 1, * FROM josm.source
+              UNION SELECT 4, 0, * FROM potlatch.source
+              UNION SELECT 5, 0, * FROM merkaartor.source;
 
+DROP TABLE IF EXISTS master_meta;
 CREATE TABLE master_meta (
     source_id    TEXT,
     source_name  TEXT,
@@ -22,14 +45,14 @@ CREATE TABLE master_meta (
     data_until   TEXT
 );
 
-INSERT INTO master_meta SELECT * FROM db.meta
-                  UNION SELECT * FROM wiki.meta
-                  UNION SELECT * FROM josm.meta;
+-- for backwards compatibility
+INSERT INTO master_meta SELECT * FROM db.source
+                  UNION SELECT * FROM wiki.source
+                  UNION SELECT * FROM josm.source;
 -- XXX                 UNION SELECT * FROM potlatch.meta
 -- XXX                 UNION SELECT * FROM merkaartor.meta;
 
 DROP TABLE IF EXISTS master_stats;
-
 CREATE TABLE master_stats (
     key   TEXT,
     value INT64
@@ -41,6 +64,8 @@ INSERT INTO master_stats SELECT * FROM db.stats
                    UNION SELECT * FROM potlatch.stats
                    UNION SELECT * FROM merkaartor.stats;
 
+-- ============================================================================
+
 INSERT INTO db.keys (key) SELECT DISTINCT key FROM wiki.wikipages        WHERE key NOT IN (SELECT key FROM db.keys);
 INSERT INTO db.keys (key) SELECT DISTINCT k   FROM josm.josm_style_rules WHERE k   NOT IN (SELECT key FROM db.keys);
 -- potlatch XXX
@@ -55,9 +80,9 @@ UPDATE db.keys SET in_merkaartor=1 WHERE key IN (SELECT key FROM merkaartor.keys
 -- too slow, so we drop it for now
 -- INSERT INTO db.tags (key, value) SELECT DISTINCT key, value FROM wiki.wikipages WHERE key || '=XX=' || value NOT IN (SELECT key || '=XX=' || value FROM db.tags);
 
+-- ============================================================================
 
 DROP TABLE IF EXISTS popular_keys;
-
 CREATE TABLE popular_keys (
     key           VARCHAR,
     count         INTEGER,
@@ -87,8 +112,9 @@ UPDATE popular_keys SET in_wiki=1    WHERE key IN (SELECT distinct key FROM wiki
 UPDATE popular_keys SET in_wiki_en=1 WHERE key IN (SELECT distinct key FROM wiki.wikipages WHERE lang='en');
 UPDATE popular_keys SET in_josm=1    WHERE key IN (SELECT distinct k   FROM josm.josm_style_rules);
 
-DROP TABLE IF EXISTS popular_metadata;
+-- ============================================================================
 
+DROP TABLE IF EXISTS popular_metadata;
 CREATE TABLE popular_metadata (
     keys        INTEGER,
     count_min   INTEGER,
@@ -111,9 +137,13 @@ UPDATE popular_keys SET scale_name  = 0 WHERE key LIKE '%:%';
 
 UPDATE popular_keys SET scale1 = 10 * scale_count + 8 * scale_users + 2 * scale_wiki + 1 * scale_josm + 2 * scale_name;
 
+-- ============================================================================
+
 INSERT INTO languages (code) SELECT distinct(lang) FROM wiki.wikipages WHERE lang NOT IN (SELECT code FROM languages);
 UPDATE languages SET wiki_key_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NULL);
 UPDATE languages SET wiki_tag_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NOT NULL);
 
+-- ============================================================================
+
 ANALYZE;
 
diff --git a/sources/master/update.sh b/sources/master/update.sh
index b8d63b2..f1598d9 100755
--- a/sources/master/update.sh
+++ b/sources/master/update.sh
@@ -9,12 +9,14 @@ set -e
 
 DIR=$1
 
+DATECMD='date +%Y-%m-%dT%H:%M:%S'
+
 if [ "x" = "x$DIR" ]; then
     echo "Usage: update.sh DIR"
     exit 1
 fi
 
-echo -n "Start master: "; date
+echo "`$DATECMD` Start master..."
 
 DATABASE=$DIR/taginfo-master.db
 
@@ -23,5 +25,5 @@ rm -f $DATABASE
 sqlite3 $DATABASE <languages.sql
 perl -pe "s|__DIR__|$DIR|" master.sql | sqlite3 $DATABASE
 
-echo -n "Done master: "; date
+echo "`$DATECMD` Done master."