From 10f53cfe47f11629b26d550c7deabba4cc427fb3 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Sun, 31 Oct 2010 22:20:12 +0100 Subject: First merkaartor support checkin --- sources/db/pre.sql | 4 +- sources/master/master.sql | 7 ++- sources/merkaartor/import_merkaartor.rb | 66 +++++++++++++++++++++++ sources/merkaartor/post.sql | 8 +++ sources/merkaartor/pre.sql | 92 +++++++++++++++++++++++++++++++++ sources/merkaartor/update.sh | 42 +++++++++++++++ 6 files changed, 216 insertions(+), 3 deletions(-) create mode 100755 sources/merkaartor/import_merkaartor.rb create mode 100644 sources/merkaartor/post.sql create mode 100644 sources/merkaartor/pre.sql create mode 100755 sources/merkaartor/update.sh (limited to 'sources') diff --git a/sources/db/pre.sql b/sources/db/pre.sql index 9404aaa..bfb98f6 100644 --- a/sources/db/pre.sql +++ b/sources/db/pre.sql @@ -45,6 +45,7 @@ CREATE TABLE keys ( in_wiki INTEGER DEFAULT 0, in_josm INTEGER DEFAULT 0, in_potlatch INTEGER DEFAULT 0, + in_merkaartor INTEGER DEFAULT 0, prevalent_values TEXT ); @@ -66,7 +67,8 @@ CREATE TABLE tags ( count_relations INTEGER DEFAULT 0, in_wiki INTEGER DEFAULT 0, in_josm INTEGER DEFAULT 0, - in_potlatch INTEGER DEFAULT 0 + in_potlatch INTEGER DEFAULT 0, + in_merkaartor INTEGER DEFAULT 0 ); DROP TABLE IF EXISTS keypairs; diff --git a/sources/master/master.sql b/sources/master/master.sql index d6d28af..867c6a7 100644 --- a/sources/master/master.sql +++ b/sources/master/master.sql @@ -38,9 +38,12 @@ INSERT INTO master_stats SELECT * FROM db.stats INSERT INTO db.keys (key) SELECT DISTINCT key FROM wiki.wikipages WHERE key NOT IN (SELECT key FROM db.keys); INSERT INTO db.keys (key) SELECT DISTINCT k FROM josm.josm_style_rules WHERE k NOT IN (SELECT key FROM db.keys); -- potlatch XXX -UPDATE db.keys SET in_wiki=1 WHERE key IN (SELECT key FROM wiki.wikipages); -UPDATE db.keys SET in_josm=1 WHERE key IN (SELECT k FROM josm.josm_style_rules); +-- INSERT INTO db.keys (key) SELECT DISTINCT key FROM merkaartor.keys WHERE key NOT IN (SELECT key FROM db.keys); + +UPDATE db.keys SET in_wiki=1 WHERE key IN (SELECT key FROM wiki.wikipages); +UPDATE db.keys SET in_josm=1 WHERE key IN (SELECT k FROM josm.josm_style_rules); -- potlatch XXX +UPDATE db.keys SET in_merkaartor=1 WHERE key IN (SELECT key FROM merkaartor.keys); -- too slow, so we drop it for now -- INSERT INTO db.tags (key, value) SELECT DISTINCT key, value FROM wiki.wikipages WHERE key || '=XX=' || value NOT IN (SELECT key || '=XX=' || value FROM db.tags); diff --git a/sources/merkaartor/import_merkaartor.rb b/sources/merkaartor/import_merkaartor.rb new file mode 100755 index 0000000..52af543 --- /dev/null +++ b/sources/merkaartor/import_merkaartor.rb @@ -0,0 +1,66 @@ +#!/usr/bin/ruby +#------------------------------------------------------------------------------ +# +# Taginfo source: Merkaartor +# +# import_merkaartor.rb +# +#------------------------------------------------------------------------------ +# +# Copyright (C) 2010 Jochen Topf +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#------------------------------------------------------------------------------ + +require 'pp' +require 'sqlite3' +require 'rexml/document' + +dir = ARGV[0] || '.' + +db = SQLite3::Database.new(dir + '/taginfo-merkaartor.db') + +db.execute('BEGIN TRANSACTION'); + +template = 'default' +db.execute('INSERT INTO templates (name) VALUES (?)', template) + +file = File.new(dir + '/git-source/Templates/' + template + '.mat') +doc = REXML::Document.new(file) + +doc.elements.each('/templates/widgets/widget') do |widget| + key = widget.attributes['tag'] + link = widget.elements['link'].attributes['src'] if widget.elements['link'] + selector = widget.elements['selector'].attributes['expr'] if widget.elements['selector'] + db.execute('INSERT INTO keys (template, key, tag_type, link, selector) VALUES (?, ?, ?, ?, ?)', template, key, widget.attributes['type'], link, selector) + widget.elements.each('description') do |desc| + db.execute('INSERT INTO key_descriptions (template, key, lang, description) VALUES (?, ?, ?, ?)', template, key, desc.attributes['locale'], desc.text) + end + widget.elements.each('value') do |valelement| + value = valelement.attributes['tag'] + vlink = valelement.elements['link'].attributes['src'] if valelement.elements['link'] + db.execute('INSERT INTO tags (template, key, value, link) VALUES (?, ?, ?, ?)', template, key, value, vlink) + widget.elements.each('description') do |desc| + db.execute('INSERT INTO tag_descriptions (template, key, value, lang, description) VALUES (?, ?, ?, ?, ?)', template, key, value, desc.attributes['locale'], desc.text) + end + end +end + + +db.execute('COMMIT'); + + +#-- THE END ------------------------------------------------------------------- diff --git a/sources/merkaartor/post.sql b/sources/merkaartor/post.sql new file mode 100644 index 0000000..9e7fea9 --- /dev/null +++ b/sources/merkaartor/post.sql @@ -0,0 +1,8 @@ +-- +-- Taginfo source: Merkaartor +-- +-- post.sql +-- + +.bail ON + diff --git a/sources/merkaartor/pre.sql b/sources/merkaartor/pre.sql new file mode 100644 index 0000000..60ba309 --- /dev/null +++ b/sources/merkaartor/pre.sql @@ -0,0 +1,92 @@ +-- +-- Taginfo source: Merkaartor +-- +-- pre.sql +-- + +.bail ON + +DROP TABLE IF EXISTS meta; + +CREATE TABLE meta ( + source_id TEXT, + source_name TEXT, + update_start TEXT, + update_end TEXT, + data_until TEXT +); + +INSERT INTO meta (source_id, source_name, update_start, data_until) SELECT 'potlatch', 'Potlatch', datetime('now'), datetime('now'); + +DROP TABLE IF EXISTS stats; + +CREATE TABLE stats ( + key TEXT, + value INT64 +); + + +-- +-- templates +-- + +DROP TABLE IF EXISTS templates; + +CREATE TABLE templates ( + name TEXT +); + +-- +-- keys +-- + +DROP TABLE IF EXISTS keys; + +CREATE TABLE keys ( + template TEXT, + key TEXT, + tag_type TEXT, + link TEXT, + selector TEXT +); + +-- +-- key_descriptions +-- + +DROP TABLE IF EXISTS key_descriptions; + +CREATE TABLE key_descriptions ( + template TEXT, + key TEXT, + lang TEXT, + description TEXT +); + +-- +-- tags +-- + +DROP TABLE IF EXISTS tags; + +CREATE TABLE tags ( + template TEXT, + key TEXT, + value TEXT, + link TEXT +); + +-- +-- tag_descriptions +-- + +DROP TABLE IF EXISTS tag_descriptions; + +CREATE TABLE tag_descriptions ( + template TEXT, + key TEXT, + value TEXT, + lang TEXT, + description TEXT +); + diff --git a/sources/merkaartor/update.sh b/sources/merkaartor/update.sh new file mode 100755 index 0000000..ae97d45 --- /dev/null +++ b/sources/merkaartor/update.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# +# Taginfo source: Merkaartor +# +# update.sh DIR +# + +set -e + +DIR=$1 + +if [ "x" = "x$DIR" ]; then + echo "Usage: update.sh DIR" + exit 1 +fi + +echo -n "Start merkaartor: "; date + +DATABASE=$DIR/taginfo-merkaartor.db + +rm -f $DATABASE + +echo "Getting resources..." +if [ -d $DIR/git-source ]; then + cd $DIR/git-source + git pull + cd - +else + git clone http://git.gitorious.org/merkaartor/main.git $DIR/git-source +fi + +echo "Running pre.sql..." +sqlite3 $DATABASE