diff options
author | Jochen Topf <jochen@topf.org> | 2010-11-13 21:31:39 +0100 |
---|---|---|
committer | Jochen Topf <jochen@topf.org> | 2010-11-13 21:31:39 +0100 |
commit | 9db63dc2526bf0b2a94e7a6f37a0f7dabdb8f12d (patch) | |
tree | 0d2ae6de413c39c21d30aaf296a98ee37977ab45 | |
parent | c82bbe69572398d44187ca7b57b1ce572f435d65 (diff) | |
download | taginfo-9db63dc2526bf0b2a94e7a6f37a0f7dabdb8f12d.tar taginfo-9db63dc2526bf0b2a94e7a6f37a0f7dabdb8f12d.tar.gz |
Multiple changes
* Improved and cleaned up source import scripts
* Improved some reports and report overview
* New osmium version that uses a better hash function
* Some experimental magic to include Javascipt in an easier way (tested in keys.erb)
32 files changed, 443 insertions, 238 deletions
diff --git a/sources/db/osmium_tagstats b/sources/db/osmium_tagstats Binary files differindex 618ee6d..ab5e21e 100755 --- a/sources/db/osmium_tagstats +++ b/sources/db/osmium_tagstats diff --git a/sources/db/post.sql b/sources/db/post.sql index e182a28..e824c48 100644 --- a/sources/db/post.sql +++ b/sources/db/post.sql @@ -61,5 +61,5 @@ UPDATE keys SET prevalent_values=( ANALYZE; -UPDATE meta SET update_end=datetime('now'); +UPDATE source SET update_end=datetime('now'); diff --git a/sources/db/pre.sql b/sources/db/pre.sql index 998b20e..38ec051 100644 --- a/sources/db/pre.sql +++ b/sources/db/pre.sql @@ -6,24 +6,7 @@ .bail ON -DROP TABLE IF EXISTS meta; - -CREATE TABLE meta ( - source_id TEXT, - source_name TEXT, - update_start TEXT, - update_end TEXT, - data_until TEXT -); - -INSERT INTO meta (source_id, source_name, update_start) SELECT 'db', 'Database', datetime('now'); - -DROP TABLE IF EXISTS stats; - -CREATE TABLE stats ( - key TEXT, - value INT64 -); +INSERT INTO source (id, name, update_start) SELECT 'db', 'Database', datetime('now'); DROP TABLE IF EXISTS keys; diff --git a/sources/db/update.sh b/sources/db/update.sh index 4ab2856..56a7652 100755 --- a/sources/db/update.sh +++ b/sources/db/update.sh @@ -9,32 +9,37 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start db: "; date +echo "`$DATECMD` Start db..." DATABASE=$DIR/taginfo-db.db rm -f $DATABASE rm -f $DIR/count.db -echo "Running pre.sql..." +echo "`$DATECMD` Running init.sql..." +sqlite3 $DATABASE <../init.sql + +echo "`$DATECMD` Running pre.sql..." sqlite3 $DATABASE <pre.sql -echo -n "Running count... "; date +echo "`$DATECMD` Running count... " HERE=`pwd` cd $DIR bzcat $DIR/planet.osm.bz2 | $HERE/osmium_tagstats - cd $HERE -echo -n "Running update_characters... "; date +echo "`$DATECMD` Running update_characters... " ./update_characters.pl $DIR -echo -n "Running post.sql... "; date +echo "`$DATECMD` Running post.sql... " perl -pe "s|__DIR__|$DIR|" post.sql | sqlite3 $DATABASE -echo -n "Done db: "; date +echo "`$DATECMD` Done db." diff --git a/sources/init.sql b/sources/init.sql new file mode 100644 index 0000000..e188915 --- /dev/null +++ b/sources/init.sql @@ -0,0 +1,29 @@ +-- +-- init.sql +-- +-- Initialize source database +-- + +.bail ON + +-- +-- Contains metadata about this source. +-- +DROP TABLE IF EXISTS source; +CREATE TABLE source ( + id TEXT, + name TEXT, + update_start TEXT, + update_end TEXT, + data_until TEXT +); + +-- +-- Contains general statistical data for this source. +-- +DROP TABLE IF EXISTS stats; +CREATE TABLE stats ( + key TEXT, + value INT64 +); + diff --git a/sources/josm/post.sql b/sources/josm/post.sql index 352b026..8dd6972 100644 --- a/sources/josm/post.sql +++ b/sources/josm/post.sql @@ -18,5 +18,5 @@ INSERT INTO stats (key, value) SELECT 'josm_tags_in_style_rules', count(distinct ANALYZE; -UPDATE meta SET update_end=datetime('now'); +UPDATE source SET update_end=datetime('now'); diff --git a/sources/josm/pre.sql b/sources/josm/pre.sql index 42a4213..5c2975d 100644 --- a/sources/josm/pre.sql +++ b/sources/josm/pre.sql @@ -6,24 +6,7 @@ .bail ON -DROP TABLE IF EXISTS meta; - -CREATE TABLE meta ( - source_id TEXT, - source_name TEXT, - update_start TEXT, - update_end TEXT, - data_until TEXT -); - -INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'josm', 'JOSM', datetime('now'), datetime('now'); - -DROP TABLE IF EXISTS stats; - -CREATE TABLE stats ( - key TEXT, - value INT64 -); +INSERT INTO source (id, name, update_start, data_until) SELECT 'josm', 'JOSM', datetime('now'), datetime('now'); -- -- josm_styles diff --git a/sources/josm/update.sh b/sources/josm/update.sh index 6eca722..abfb7e9 100755 --- a/sources/josm/update.sh +++ b/sources/josm/update.sh @@ -9,12 +9,14 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start josm: "; date +echo "`$DATECMD` Start josm..." DATABASE=$DIR/taginfo-josm.db ELEMSTYLES=$DIR/elemstyles.xml @@ -22,17 +24,20 @@ ELEMSTYLES=$DIR/elemstyles.xml rm -f $DATABASE rm -f $ELEMSTYLES -echo "Getting styles..." +echo "`$DATECMD` Getting styles..." wget -O $ELEMSTYLES http://josm.openstreetmap.de/svn/trunk/styles/standard/elemstyles.xml -echo "Running pre.sql..." +echo "`$DATECMD` Running init.sql..." +sqlite3 $DATABASE <../init.sql + +echo "`$DATECMD` Running pre.sql..." sqlite3 $DATABASE <pre.sql -echo "Running import..." +echo "`$DATECMD` Running import..." ./import_josm.rb $DIR -echo "Running post.sql..." +echo "`$DATECMD` Running post.sql..." sqlite3 $DATABASE <post.sql -echo -n "Done josm: "; date +echo "`$DATECMD` Done josm." diff --git a/sources/master/languages.sql b/sources/master/languages.sql index 968e689..eb2b6d9 100644 --- a/sources/master/languages.sql +++ b/sources/master/languages.sql @@ -6,15 +6,17 @@ .bail ON +-- +-- Contains all the languages Taginfo knows about. +-- DROP TABLE IF EXISTS languages; - CREATE TABLE languages ( code VARCHAR, - iso639_1 VARCHAR, + iso639_1 VARCHAR, -- official ISO 639-1 code (if available) english_name VARCHAR, native_name VARCHAR, - wiki_key_pages INTEGER, - wiki_tag_pages INTEGER + wiki_key_pages INTEGER, -- count of wiki pages with the title "code:Key:*" (or "Key:*" for code='en') + wiki_tag_pages INTEGER -- count of wiki pages with the title "code:Tag:*" (or "Tag:*" for code='en') ); INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('ar', 'ar', 'Arabic', 'العربية'); @@ -48,5 +50,3 @@ INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh', INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('zh-hans', NULL, 'Chinese', '中文'); -- INSERT INTO languages (code, iso639_1, english_name, native_name) VALUES ('', '', '', ''); -ANALYZE; - diff --git a/sources/master/master.sql b/sources/master/master.sql index fc0ccef..c559426 100644 --- a/sources/master/master.sql +++ b/sources/master/master.sql @@ -1,8 +1,10 @@ +-- ============================================================================ -- -- Taginfo -- -- master.sql -- +-- ============================================================================ .bail ON @@ -12,8 +14,29 @@ ATTACH DATABASE '__DIR__/josm/taginfo-josm.db' AS josm; ATTACH DATABASE '__DIR__/potlatch/taginfo-potlatch.db' AS potlatch; ATTACH DATABASE '__DIR__/merkaartor/taginfo-merkaartor.db' AS merkaartor; -DROP TABLE IF EXISTS master_meta; +-- ============================================================================ + +-- +-- Collects information about all the sources. +-- +DROP TABLE IF EXISTS sources; +CREATE TABLE sources ( + no INTEGER, + visible INTEGER, + id TEXT, + name TEXT, + update_start TEXT, + update_end TEXT, + data_until TEXT +); + +INSERT INTO sources SELECT 1, 1, * FROM db.source + UNION SELECT 2, 1, * FROM wiki.source + UNION SELECT 3, 1, * FROM josm.source + UNION SELECT 4, 0, * FROM potlatch.source + UNION SELECT 5, 0, * FROM merkaartor.source; +DROP TABLE IF EXISTS master_meta; CREATE TABLE master_meta ( source_id TEXT, source_name TEXT, @@ -22,14 +45,14 @@ CREATE TABLE master_meta ( data_until TEXT ); -INSERT INTO master_meta SELECT * FROM db.meta - UNION SELECT * FROM wiki.meta - UNION SELECT * FROM josm.meta; +-- for backwards compatibility +INSERT INTO master_meta SELECT * FROM db.source + UNION SELECT * FROM wiki.source + UNION SELECT * FROM josm.source; -- XXX UNION SELECT * FROM potlatch.meta -- XXX UNION SELECT * FROM merkaartor.meta; DROP TABLE IF EXISTS master_stats; - CREATE TABLE master_stats ( key TEXT, value INT64 @@ -41,6 +64,8 @@ INSERT INTO master_stats SELECT * FROM db.stats UNION SELECT * FROM potlatch.stats UNION SELECT * FROM merkaartor.stats; +-- ============================================================================ + INSERT INTO db.keys (key) SELECT DISTINCT key FROM wiki.wikipages WHERE key NOT IN (SELECT key FROM db.keys); INSERT INTO db.keys (key) SELECT DISTINCT k FROM josm.josm_style_rules WHERE k NOT IN (SELECT key FROM db.keys); -- potlatch XXX @@ -55,9 +80,9 @@ UPDATE db.keys SET in_merkaartor=1 WHERE key IN (SELECT key FROM merkaartor.keys -- too slow, so we drop it for now -- INSERT INTO db.tags (key, value) SELECT DISTINCT key, value FROM wiki.wikipages WHERE key || '=XX=' || value NOT IN (SELECT key || '=XX=' || value FROM db.tags); +-- ============================================================================ DROP TABLE IF EXISTS popular_keys; - CREATE TABLE popular_keys ( key VARCHAR, count INTEGER, @@ -87,8 +112,9 @@ UPDATE popular_keys SET in_wiki=1 WHERE key IN (SELECT distinct key FROM wiki UPDATE popular_keys SET in_wiki_en=1 WHERE key IN (SELECT distinct key FROM wiki.wikipages WHERE lang='en'); UPDATE popular_keys SET in_josm=1 WHERE key IN (SELECT distinct k FROM josm.josm_style_rules); -DROP TABLE IF EXISTS popular_metadata; +-- ============================================================================ +DROP TABLE IF EXISTS popular_metadata; CREATE TABLE popular_metadata ( keys INTEGER, count_min INTEGER, @@ -111,9 +137,13 @@ UPDATE popular_keys SET scale_name = 0 WHERE key LIKE '%:%'; UPDATE popular_keys SET scale1 = 10 * scale_count + 8 * scale_users + 2 * scale_wiki + 1 * scale_josm + 2 * scale_name; +-- ============================================================================ + INSERT INTO languages (code) SELECT distinct(lang) FROM wiki.wikipages WHERE lang NOT IN (SELECT code FROM languages); UPDATE languages SET wiki_key_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NULL); UPDATE languages SET wiki_tag_pages=(SELECT count(distinct key) FROM wiki.wikipages WHERE lang=code AND value IS NOT NULL); +-- ============================================================================ + ANALYZE; diff --git a/sources/master/update.sh b/sources/master/update.sh index b8d63b2..f1598d9 100755 --- a/sources/master/update.sh +++ b/sources/master/update.sh @@ -9,12 +9,14 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start master: "; date +echo "`$DATECMD` Start master..." DATABASE=$DIR/taginfo-master.db @@ -23,5 +25,5 @@ rm -f $DATABASE sqlite3 $DATABASE <languages.sql perl -pe "s|__DIR__|$DIR|" master.sql | sqlite3 $DATABASE -echo -n "Done master: "; date +echo "`$DATECMD` Done master." diff --git a/sources/merkaartor/post.sql b/sources/merkaartor/post.sql index 9e7fea9..06bcef0 100644 --- a/sources/merkaartor/post.sql +++ b/sources/merkaartor/post.sql @@ -6,3 +6,7 @@ .bail ON +ANALYZE; + +UPDATE source SET update_end=datetime('now'); + diff --git a/sources/merkaartor/pre.sql b/sources/merkaartor/pre.sql index 60ba309..76d8012 100644 --- a/sources/merkaartor/pre.sql +++ b/sources/merkaartor/pre.sql @@ -6,25 +6,7 @@ .bail ON -DROP TABLE IF EXISTS meta; - -CREATE TABLE meta ( - source_id TEXT, - source_name TEXT, - update_start TEXT, - update_end TEXT, - data_until TEXT -); - -INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now'); - -DROP TABLE IF EXISTS stats; - -CREATE TABLE stats ( - key TEXT, - value INT64 -); - +INSERT INTO source (id, name, update_start, data_until) SELECT 'merkaartor', 'Merkaartor', datetime('now'), datetime('now'); -- -- templates diff --git a/sources/merkaartor/update.sh b/sources/merkaartor/update.sh index ae97d45..a112b27 100755 --- a/sources/merkaartor/update.sh +++ b/sources/merkaartor/update.sh @@ -9,18 +9,20 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start merkaartor: "; date +echo "`$DATECMD` Start merkaartor..." DATABASE=$DIR/taginfo-merkaartor.db rm -f $DATABASE -echo "Getting resources..." +echo "`$DATECMD` Updating resources..." if [ -d $DIR/git-source ]; then cd $DIR/git-source git pull @@ -29,14 +31,17 @@ else git clone http://git.gitorious.org/merkaartor/main.git $DIR/git-source fi -echo "Running pre.sql..." +echo "`$DATECMD` Running init.sql..." +sqlite3 $DATABASE <../init.sql + +echo "`$DATECMD` Running pre.sql..." sqlite3 $DATABASE <pre.sql -echo "Running import..." +echo "`$DATECMD` Running import..." ./import_merkaartor.rb $DIR -echo "Running post.sql..." +echo "`$DATECMD` Running post.sql..." sqlite3 $DATABASE <post.sql -echo -n "Done merkaartor: "; date +echo "`$DATECMD` Done merkaartor." diff --git a/sources/potlatch/post.sql b/sources/potlatch/post.sql index 3446d08..1254b89 100644 --- a/sources/potlatch/post.sql +++ b/sources/potlatch/post.sql @@ -11,3 +11,5 @@ UPDATE features SET category_name = (SELECT name FROM categories WHERE id=catego ANALYZE; +UPDATE source SET update_end=datetime('now'); + diff --git a/sources/potlatch/pre.sql b/sources/potlatch/pre.sql index 9f79445..35d8c9d 100644 --- a/sources/potlatch/pre.sql +++ b/sources/potlatch/pre.sql @@ -6,24 +6,7 @@ .bail ON -DROP TABLE IF EXISTS meta; - -CREATE TABLE meta ( - source_id TEXT, - source_name TEXT, - update_start TEXT, - update_end TEXT, - data_until TEXT -); - -INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now'); - -DROP TABLE IF EXISTS stats; - -CREATE TABLE stats ( - key TEXT, - value INT64 -); +INSERT INTO source (id, name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now'); -- -- categories diff --git a/sources/potlatch/update.sh b/sources/potlatch/update.sh index 8ba9956..748ff4a 100755 --- a/sources/potlatch/update.sh +++ b/sources/potlatch/update.sh @@ -9,32 +9,37 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start potlatch: "; date +echo "`$DATECMD` Start potlatch..." DATABASE=$DIR/taginfo-potlatch.db rm -f $DATABASE -echo "Getting resources..." +echo "`$DATECMD` Updating resources..." if [ -d $DIR/resources ]; then svn update $DIR/resources else svn checkout http://svn.openstreetmap.org/applications/editors/potlatch2/resources $DIR/resources fi -echo "Running pre.sql..." +echo "`$DATECMD` Running init.sql..." +sqlite3 $DATABASE <../init.sql + +echo "`$DATECMD` Running pre.sql..." sqlite3 $DATABASE <pre.sql -echo "Running import..." +echo "`$DATECMD` Running import..." ./import_potlatch.rb $DIR -echo "Running post.sql..." +echo "`$DATECMD` Running post.sql..." sqlite3 $DATABASE <post.sql -echo -n "Done potlatch: "; date +echo "`$DATECMD` Done potlatch." diff --git a/sources/update_all.sh b/sources/update_all.sh index f7e4459..39e58ce 100755 --- a/sources/update_all.sh +++ b/sources/update_all.sh @@ -9,6 +9,8 @@ SOURCES="josm potlatch merkaartor wiki db" set -e +DATECMD='date +%Y-%m-%dT%H:%M:%S' + DIR=$1 if [ "x" = "x$DIR" ]; then @@ -16,9 +18,11 @@ if [ "x" = "x$DIR" ]; then exit 1 fi -exec >$DIR/update_all.log 2>&1 +LOGFILE=`date +%Y%m%dT%H%M` +mkdir -p $DIR/log +exec >$DIR/log/$LOGFILE.log 2>&1 -echo -n "Start: "; date +echo "`$DATECMD` Start update_all..." mkdir -p $DIR/download @@ -40,7 +44,7 @@ cd .. for source in $SOURCES; do echo "=====================================" - echo "Running bzip2..." + echo "Running bzip2 on $source..." bzip2 -9 -c $DIR/$source/taginfo-$source.db >$DIR/download/taginfo-$source.db.bz2 echo "Done." done @@ -49,5 +53,6 @@ echo "Running bzip2..." bzip2 -9 -c $DIR/taginfo-master.db >$DIR/download/taginfo-master.db.bz2 echo "Done." -echo -n "Done: "; date +echo "=====================================" +echo "`$DATECMD` Done update_all." diff --git a/sources/wiki/post.sql b/sources/wiki/post.sql index eaf9ec0..773a04d 100644 --- a/sources/wiki/post.sql +++ b/sources/wiki/post.sql @@ -36,5 +36,5 @@ INSERT INTO stats (key, value) SELECT 'wiki_languages', count(*) FROM wiki_langu ANALYZE; -UPDATE meta SET update_end=datetime('now'); +UPDATE source SET update_end=datetime('now'); diff --git a/sources/wiki/pre.sql b/sources/wiki/pre.sql index 0e8200e..56e9538 100644 --- a/sources/wiki/pre.sql +++ b/sources/wiki/pre.sql @@ -6,24 +6,7 @@ .bail ON -DROP TABLE IF EXISTS meta; - -CREATE TABLE meta ( - source_id TEXT, - source_name TEXT, - update_start TEXT, - update_end TEXT, - data_until TEXT -); - -INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'wiki', 'Wiki', datetime('now'), datetime('now'); - -DROP TABLE IF EXISTS stats; - -CREATE TABLE stats ( - key TEXT, - value INT64 -); +INSERT INTO source (id, name, update_start, data_until) SELECT 'wiki', 'Wiki', datetime('now'), datetime('now'); DROP TABLE IF EXISTS wikipages; diff --git a/sources/wiki/update.sh b/sources/wiki/update.sh index b6f63a6..7d7dc6b 100755 --- a/sources/wiki/update.sh +++ b/sources/wiki/update.sh @@ -9,12 +9,14 @@ set -e DIR=$1 +DATECMD='date +%Y-%m-%dT%H:%M:%S' + if [ "x" = "x$DIR" ]; then echo "Usage: update.sh DIR" exit 1 fi -echo -n "Start wiki: "; date +echo "`$DATECMD` Start wiki..." DATABASE=$DIR/taginfo-wiki.db LOGFILE=$DIR/get_wiki_data.log @@ -24,17 +26,20 @@ rm -f $DIR/tagpages.list rm -f $LOGFILE rm -f $DATABASE -echo "Running pre.sql..." +echo "`$DATECMD` Running init.sql..." +sqlite3 $DATABASE <../init.sql + +echo "`$DATECMD` Running pre.sql..." sqlite3 $DATABASE <pre.sql -echo "Getting page list..." +echo "`$DATECMD` Getting page list..." ./get_page_list.rb $DIR -echo "Getting wiki data..." +echo "`$DATECMD` Getting wiki data..." ./get_wiki_data.rb $DIR >$LOGFILE -echo "Running post.sql..." +echo "`$DATECMD` Running post.sql..." sqlite3 $DATABASE <post.sql -echo -n "Done wiki: "; date +echo "`$DATECMD` Done wiki." diff --git a/web/lib/javascript.rb b/web/lib/javascript.rb new file mode 100644 index 0000000..7f43d44 --- /dev/null +++ b/web/lib/javascript.rb @@ -0,0 +1,88 @@ +# web/lib/javascript.rb + +def javascript(url=nil, &block) + @javascript ||= Array.new + @javascript << Javascript.new(url, &block) +end + +def javascript_tags + @javascript.map{ |js| js.to_html } +end + +class Javascript + + def initialize(file) + if file.nil? + c = '' + r = yield c + @content = c + r + else + @file = file + end + end + + def to_html + if @file.nil? + %Q{ <script type="text/javascript">//<![CDATA[\n#{ @content }//]]></script>\n} + else + %Q{ <script type="text/javascript" src="/js/#{ @file }.js"></script>\n} + end + end + +end + +class JQuery + + # "include" the convenience methods from R18n::Helpers. + # Uses extend instead of include, because we want this + # to work not with instances of JQuery but they should + # show up as JQuery class methods. + extend R18n::Helpers + + def self.ready + "jQuery(document).ready(function() {\n" + + yield + + "});\n" + end + + def self.flexigrid(id, options) + defaults = { + :method => 'GET', + :dataType => 'json', + :pagetext => t.flexigrid.pagetext, + :pagestat => t.flexigrid.pagestat, + :outof => t.flexigrid.outof, + :findtext => t.flexigrid.findtext, + :procmsg => t.flexigrid.procmsg, + :nomsg => t.flexigrid.nomsg, + :errormsg => t.flexigrid.errormsg, + :showToggleBtn => false, + :usepager => true, + :useRp => true, + :rp => 15, + :rpOptions => [10,15,20,25,50,100], + } + "jQuery('##{id}').flexigrid(" + defaults.merge(options).to_json + ");\n" + end + +end + +class JS + + # + # Careful, deep magic! + # + # We redefine the to_json method of the String argument to return + # the raw string. This way we can do JS.raw("foo").to_json and get "foo". + # + def self.raw(code) + code.instance_eval do + def to_json + to_s + end + end + code + end + +end + diff --git a/web/lib/reports.rb b/web/lib/reports.rb index 99d500a..0895359 100644 --- a/web/lib/reports.rb +++ b/web/lib/reports.rb @@ -11,15 +11,19 @@ class Report end end - def initialize(title, sources) + def initialize(title, *sources) @@reports << self @title = title @sources = Hash.new - sources.each do |s| - @sources[s] = 1 + sources.each do |id| + @sources[id] = 1 end end + def uses_source?(id) + sources.has_key? id + end + def name @title.gsub(/ /, '_').downcase end @@ -30,9 +34,10 @@ class Report end -Report.new 'Characters in Keys', %w(db) -Report.new 'Frequently Used Keys Without Wiki Page', %w(db wiki) -Report.new 'Key Lengths', %w(db) -Report.new 'Language Comparison Table for Keys in the Wiki', %w(wiki) -Report.new 'Languages', %w(wiki) -Report.new 'Wiki Pages About Non-Existing Keys', %w(db wiki) +Report.new 'Characters in Keys', :db +Report.new 'Frequently Used Keys Without Wiki Page', :db, :wiki +Report.new 'Key Lengths', :db +Report.new 'Language Comparison Table for Keys in the Wiki', :wiki +Report.new 'Languages', :wiki +Report.new 'Wiki Pages About Non-Existing Keys', :db, :wiki + diff --git a/web/lib/sources.rb b/web/lib/sources.rb new file mode 100644 index 0000000..918dd4d --- /dev/null +++ b/web/lib/sources.rb @@ -0,0 +1,57 @@ +# web/lib/sources.rb +class Source + + @@sources = Array.new + + attr_reader :id, :name, :data_until, :update_start, :update_end + + # Enumerate all available sources + def self.each + @@sources.each do |source| + yield source + end + end + + # The number of available sources + def self.size + @@sources.size + end + + # Create new source + # id - Symbol with id for this source + # name - Name of this source + def initialize(id, name, data_until, update_start, update_end) + @id = id.to_sym + @name = name + @data_until = data_until + @update_start = update_start + @update_end = update_end + @@sources << self + end + + # The URL where this source is described + def url + "/sources/#{ @id }" + end + + # The img URL of this source + def imgurl(size=16) + "/img/sources/#{ @id }.#{ size }.png" + end + + # Returns img tag for this source + def img(size=16) + %Q{<img src="#{ imgurl(size) }" alt="#{ name }" title="#{ name }" width="#{ size }" height="#{ size }"/>} + end + + # Returns a link to this source + def link_img + %Q{<a href="#{ url }">#{ img }</a>} + end + + # Returns a link to this source + def link_name + %Q{<a href="#{ url }">#{ name }</a>} + end + +end diff --git a/web/public/css/taginfo.css b/web/public/css/taginfo.css index c125799..b5d69e8 100644 --- a/web/public/css/taginfo.css +++ b/web/public/css/taginfo.css @@ -28,8 +28,8 @@ p { margin-top: 2px; } -dd { - padding-bottom: 12px; +p.text { + max-width: 60em; } iframe { @@ -311,6 +311,7 @@ table.list { table.list th { padding: 4px; + font-weight: normal; } table.list td { @@ -325,6 +326,44 @@ table.list td.border { /* ========== */ +table.reports { + border-collapse: collapse; +} + +table.reports th.source { + background-color: #f0f0f0; + font-size: 80%; + padding-bottom: 4px; + -moz-border-radius: 4px 4px 0 0; + -khtml-border-radius: 4px 4px 0 0; + -webkit-border-radius: 4px 4px 0 0; + -chrome-border-radius: 4px 4px 0 0; + -o-border-radius: 4px 4px 0 0; +} + +table.reports td { + padding: 2px 4px; +} + +table.reports td.source { + background-color: #f0f0f0; + text-align: center; + padding: 2px 0 0 0; + width: 24px; +} + +table.reports th.last { + background-color: #f0f0f0; + height: 8px; + -moz-border-radius: 0 0 4px 4px; + -khtml-border-radius: 0 0 4px 4px; + -webkit-border-radius: 0 0 4px 4px; + -chrome-border-radius: 0 0 4px 4px; + -o-border-radius: 0 0 4px 4px; +} + +/* ========== */ + div#canvas-values { position: absolute; border: 1px solid #cccccc; diff --git a/web/public/img/sources/wiki/wiki-e.png b/web/public/img/sources/wiki/wiki-e.png Binary files differindex 63e468e..1ec9fb0 100644 --- a/web/public/img/sources/wiki/wiki-e.png +++ b/web/public/img/sources/wiki/wiki-e.png diff --git a/web/taginfo.rb b/web/taginfo.rb index 8bc5233..a3de09b 100755 --- a/web/taginfo.rb +++ b/web/taginfo.rb @@ -35,8 +35,10 @@ require 'sinatra/base' require 'sinatra/r18n' require 'lib/utils.rb' +require 'lib/javascript.rb' require 'lib/language.rb' require 'lib/sql.rb' +require 'lib/sources.rb' require 'lib/reports.rb' #------------------------------------------------------------------------------ @@ -45,6 +47,15 @@ TAGCLOUD_NUMBER_OF_TAGS = 200 #------------------------------------------------------------------------------ +db = SQL::Database.new('../../data') + +# XXX update this to use new sources table +db.select('SELECT * FROM master_meta ORDER BY source_name').execute().each do |source| + Source.new source['source_id'], source['source_name'], source['data_until'], source['update_start'], source['update_end'] +end + +db.close + class Taginfo < Sinatra::Base register Sinatra::R18n @@ -84,6 +95,12 @@ class Taginfo < Sinatra::Base # end params[:locale] = 'en' + javascript 'jquery-1.3.2.min' + javascript 'jquery-ui-1.7.2.all.min' + javascript 'flexigrid-minified' + javascript 'taginfo' + javascript 'protovis-r3.2' + @db = SQL::Database.new('../../data') @data_until = @db.select("SELECT min(data_until) FROM master_meta").get_first_value().sub(/:..$/, '') diff --git a/web/views/keys.erb b/web/views/keys.erb index ced0a96..e398fa0 100644 --- a/web/views/keys.erb +++ b/web/views/keys.erb @@ -3,63 +3,48 @@ <table id="keys"> </table> -<script type="text/javascript"> - jQuery(document).ready(function() { - jQuery('#keys').flexigrid({ - url: '/api/2/db/keys', - method: 'GET', - dataType: 'json', - colModel: [ - { display: 'Key', name: 'key', width: 180, sortable: true, align: 'left' }, - { display: '<span title="Number of objects with this key"><img src="/img/types/all.16.png" alt=""/> Total</span>', name: 'count_all', width: 250, sortable: true, align: 'center' }, - { display: '<span title="Number of nodes with this key"><img src="/img/types/node.16.png" alt=""/> Nodes</span>', name: 'count_nodes', width: 250, sortable: true, align: 'center' }, - { display: '<span title="Number of ways with this key"><img src="/img/types/way.16.png" alt=""/> Ways</span>', name: 'count_ways', width: 250, sortable: true, align: 'center' }, - { display: '<span title="Number of relations with this key"><img src="/img/types/relation.16.png" alt=""/> Relation</span>', name: 'count_relations', width: 250, sortable: true, align: 'center' }, - { display: 'Users', name: 'users_all', width: 44, sortable: true, align: 'right' }, - { display: '<img src="/img/sources/wiki.16.png" alt="Wiki" title="Wiki"/>', name: 'in_wiki', width: 20, sortable: true, align: 'center' }, - { display: '<img src="/img/sources/josm.16.png" alt="JOSM" title="JOSM"/>', name: 'in_josm', width: 20, sortable: true, align: 'center' }, - // { display: '<img src="/img/sources/potlatch.16.png" alt="Potlatch 2" title="Potlatch 2"/>', name: 'in_potlatch', width: 20, sortable: true, align: 'center' }, - // { display: '<img src="/img/sources/merkaartor.16.png" alt="Merkaartor" title="Merkaartor"/>', name: 'in_merkaartor', width: 20, sortable: true, align: 'center' }, - { display: '<span title="Number of different values for this key">Values</span>', name: 'values_all', width: 70, sortable: true, align: 'right' }, - { display: 'Prevalent Values', name: 'prevalent_values', width: 500, sortable: true, align: 'left' } +<% javascript do + JQuery.ready do + JQuery.flexigrid(:keys, + :url => '/api/2/db/keys', + :colModel => [ + { :display => 'Key', :name => 'key', :width => 180, :sortable => true, :align => 'left' }, + { :display => '<span title="Number of objects with this key"><img src="/img/types/all.16.png" alt=""/> Total</span>', :name => 'count_all', :width => 250, :sortable => true, :align => 'center' }, + { :display => '<span title="Number of nodes with this key"><img src="/img/types/node.16.png" alt=""/> Nodes</span>', :name => 'count_nodes', :width => 250, :sortable => true, :align => 'center' }, + { :display => '<span title="Number of ways with this key"><img src="/img/types/way.16.png" alt=""/> Ways</span>', :name => 'count_ways', :width => 250, :sortable => true, :align => 'center' }, + { :display => '<span title="Number of relations with this key"><img src="/img/types/relation.16.png" alt=""/> Relation</span>', :name => 'count_relations', :width => 250, :sortable => true, :align => 'center' }, + { :display => 'Users', :name => 'users_all', :width => 44, :sortable => true, :align => 'right' }, + { :display => '<img src="/img/sources/wiki.16.png" alt="Wiki" title="Wiki"/>', :name => 'in_wiki', :width => 20, :sortable => true, :align => 'center' }, + { :display => '<img src="/img/sources/josm.16.png" alt="JOSM" title="JOSM"/>', :name => 'in_josm', :width => 20, :sortable => true, :align => 'center' }, + { :display => '<span title="Number of different values for this key">Values</span>', :name => 'values_all', :width => 70, :sortable => true, :align => 'right' }, + { :display => 'Prevalent Values', :name => 'prevalent_values', :width => 500, :sortable => true, :align => 'left' } ], - searchitems: [ - { display: 'Key', name: 'key' } + :searchitems => [ + { :display => 'Key', :name => 'key' } ], - sortname: 'count_all', - sortorder: 'desc', - pagetext: '<%= t.flexigrid.pagetext %>', - pagestat: '<%= t.flexigrid.pagestat %>', - outof: '<%= t.flexigrid.outof %>', - findtext: '<%= t.flexigrid.findtext %>', - procmsg: '<%= t.flexigrid.procmsg %>', - nomsg: '<%= t.flexigrid.nomsg %>', - errormsg: '<%= t.flexigrid.errormsg %>', - showToggleBtn: false, - usepager: true, - useRp: true, - rp: 15, - rpOptions: [10,15,20,25,50,100], - height: 420, - preProcess: function(data) { - data.rows = jQuery.map(data.data, function(row, i) { - return { 'cell': [ - link_to_key(row.key), - print_value_with_percent(row.count_all, row.count_all_fraction), - print_value_with_percent(row.count_nodes, row.count_nodes_fraction), - print_value_with_percent(row.count_ways, row.count_ways_fraction), - print_value_with_percent(row.count_relations, row.count_relations_fraction), - print_with_ts(row.users_all), - row.in_wiki ? '✔' : '-', - row.in_josm ? '✔' : '-', - // row.in_potlatch ? '✔' : '-', - // row.in_merkaartor ? '✔' : '-', - print_with_ts(row.values_all), - print_tag_list(row.key, row.prevalent_values) - ] }; - }); - return data; - } - }); + :sortname => 'count_all', + :sortorder => 'desc', + :height => 420, + :preProcess => JS.raw(%q{ +function(data) { + data.rows = jQuery.map(data.data, function(row, i) { + return { 'cell': [ + link_to_key(row.key), + print_value_with_percent(row.count_all, row.count_all_fraction), + print_value_with_percent(row.count_nodes, row.count_nodes_fraction), + print_value_with_percent(row.count_ways, row.count_ways_fraction), + print_value_with_percent(row.count_relations, row.count_relations_fraction), + print_with_ts(row.users_all), + row.in_wiki ? '✔' : '-', + row.in_josm ? '✔' : '-', + print_with_ts(row.values_all), + print_tag_list(row.key, row.prevalent_values) + ] }; }); -</script> + return data; +} + }) + ) + end +end +%> diff --git a/web/views/layout.erb b/web/views/layout.erb index e2fc109..ebbe352 100644 --- a/web/views/layout.erb +++ b/web/views/layout.erb @@ -5,13 +5,9 @@ <meta http-equiv="content-type" content="text/html; charset=utf-8" /> <meta name="content-language" content="<%= r18n.locale.code %>" /> <link rel="stylesheet" type="text/css" href="/css/taginfo.css" /> - <link rel="Stylesheet" type="text/css" href="/css/smoothness/jquery-ui-1.7.3.custom.css" /> + <link rel="stylesheet" type="text/css" href="/css/smoothness/jquery-ui-1.7.3.custom.css" /> <link rel="stylesheet" type="text/css" href="/css/flexigrid/flexigrid.css" /> - <script type="text/javascript" src="/js/jquery-1.3.2.min.js"></script> - <script type="text/javascript" src="/js/jquery-ui-1.7.2.all.min.js"></script> - <script type="text/javascript" src="/js/flexigrid-minified.js"></script> - <script type="text/javascript" src="/js/taginfo.js"></script> - <script type="text/javascript" src="/js/protovis-r3.2.js"></script> +<%= javascript_tags %> </head> <body> <div id="top"> diff --git a/web/views/reports/characters_in_keys.erb b/web/views/reports/characters_in_keys.erb index b54a0a4..8f0b578 100644 --- a/web/views/reports/characters_in_keys.erb +++ b/web/views/reports/characters_in_keys.erb @@ -51,6 +51,12 @@ case latin letters (a-z) and the underscore (_) are used. Sometimes the colon <td><%= sprintf '%.2f', @db.stats('characters_in_keys_rest') * 100.0 / @db.stats('num_keys') %>%</td> <td>Everything else.</td> </tr> + <tr> + <td style="border-top: 1px solid #000000;"></td> + <td style="border-top: 1px solid #000000;"><%= @db.stats('num_keys') %></td> + <td style="border-top: 1px solid #000000;">100%</td> + <td style="border-top: 1px solid #000000;">Total</td> + </tr> </table> </div> <div id="tabs-whitespace"> @@ -90,7 +96,7 @@ case latin letters (a-z) and the underscore (_) are used. Sometimes the colon { display: 'Characters in Key', name: 'characters', width: 810, sortable: true, align: 'left' } ], width: 990, - height: 170 + height: 200 }); jQuery('#keys-whitespace').flexigrid({ url: '/api/2/db/keys?filters=characters_space', diff --git a/web/views/reports/index.erb b/web/views/reports/index.erb index b9642e8..d77a4d2 100644 --- a/web/views/reports/index.erb +++ b/web/views/reports/index.erb @@ -1,19 +1,21 @@ <h1>Reports</h1> -<p>Reports show the data from different perspectives. They often bring -together the data from several sources in interesting ways.</p> +<p class="text">Reports show the data from different perspectives. +They often bring together data from several sources in interesting ways. Some +of the reports can help with finding certain kinds of errors, such as popular +keys without wiki pages.</p> -<table> - <% Report.each do |report| %> - <tr> - <% %w(db wiki).each do |source| %> - <td> - <% if report.sources[source] %> - <img src="/img/sources/<%= source %>.16.png" alt="<%= source.capitalize %>" title="<%= source.capitalize %>" width="16" height="16"/> - <% end %> - </td> - <% end %> - <td style="padding: 2px;"><a href="<%= report.url %>"><%= report.title %></a></td> - </tr> - <% end %> +<table class="reports"> + <tr><th class="source" colspan="<%= Source.size %>"/>Data<br/>Sources</th><th></th></tr> +<% Report.each do |report| %> + <tr> +<% Source.each do |source| %> + <td class="source"><%= source.link_img if report.uses_source? source.id %></td> +<% end %> + <td><a href="<%= report.url %>"><%= report.title %></a></td> + </tr> +<% end %> + <tr><th class="last" colspan="<%= Source.size %>"/></th><th></th></tr> </table> + + diff --git a/web/views/sources/index.erb b/web/views/sources/index.erb index ba7fdb8..ee40a60 100644 --- a/web/views/sources/index.erb +++ b/web/views/sources/index.erb @@ -10,13 +10,12 @@ have to go back to the source.</p> <tr> <th></th><th>Source</th><th>Data until*</th><th>Last update run</th><th></th> </tr> - <% @sources.each do |source| %> + <% Source.each do |source| %> <tr> <td class="border"></td> - <td><a href="/sources/<%= source['source_id'] %>"><img src="/img/sources/<%= source['source_id'] %>.16.png" alt="" width="16" height="16"/></a> - <a href="/sources/<%= source['source_id'] %>"><%= source['source_name'] %></a></td> - <td><%= source['data_until'] %></td> - <td><%= source['update_start'] %> to <%= source['update_end'] %></td> + <td><%= source.link_img + ' ' + source.link_name %> + <td><%= source.data_until %></td> + <td><%= source.update_start %> to <%= source.update_end %></td> <td class="border"></td> </tr> <% end %> |