diff options
| -rwxr-xr-x | sources/languages/import_unicode_scripts.rb | 72 | ||||
| -rw-r--r-- | sources/languages/pre.sql | 15 | ||||
| -rwxr-xr-x | sources/languages/update.sh | 13 | ||||
| -rw-r--r-- | taginfo-config-example.json | 2 |
4 files changed, 100 insertions, 2 deletions
diff --git a/sources/languages/import_unicode_scripts.rb b/sources/languages/import_unicode_scripts.rb new file mode 100755 index 0000000..7855a78 --- /dev/null +++ b/sources/languages/import_unicode_scripts.rb @@ -0,0 +1,72 @@ +#!/usr/bin/ruby +#------------------------------------------------------------------------------ +# +# Taginfo source: Languages +# +# import_unicode_scripts.rb +# +#------------------------------------------------------------------------------ +# +# Copyright (C) 2013 Jochen Topf <jochen@remote.org> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +#------------------------------------------------------------------------------ + +require 'rubygems' + +require 'sqlite3' + +dir = ARGV[0] || '.' + +db = SQLite3::Database.new(dir + '/taginfo-languages.db') + +property_value_alias_file = "#{dir}/PropertyValueAliases.txt" +codepoint_script_mapping_file = "#{dir}/Scripts.txt" + +db.execute('BEGIN TRANSACTION'); + +open(property_value_alias_file) do |file| + file.each do |line| + line.chomp! + if line.match(%r{^sc ;}) + (sc, script, name) = line.split(%r{\s*;\s*}) + db.execute("INSERT INTO unicode_scripts (script, name) VALUES (?, ?)", script, name) + end + end +end + +open(codepoint_script_mapping_file) do |file| + file.each do |line| + line.chomp! + line.sub!(%r{\s*#.*}, '') + next if line.match(%r{^$}) + (codes, script) = line.split(%r{\s+;\s+}) + if codes.match(%r{^[0-9A-F]{4,5}$}) + from = codes + to = codes + elsif codes.match(%r{^([0-9A-F]{4,5})..([0-9A-F]{4,5})$}) + from = $1 + to = $2 + else + puts "Line does not match: #{line}" + next + end + db.execute("INSERT INTO unicode_codepoint_script_mapping (codepoint_from, codepoint_to, name) VALUES (?, ?, ?)", from, to, script) + end +end + +db.execute('COMMIT'); + diff --git a/sources/languages/pre.sql b/sources/languages/pre.sql index c1a505c..2cf0aa7 100644 --- a/sources/languages/pre.sql +++ b/sources/languages/pre.sql @@ -20,3 +20,18 @@ CREATE TABLE subtags ( prefix TEXT ); +DROP TABLE IF EXISTS unicode_scripts; + +CREATE TABLE unicode_scripts ( + script TEXT, + name TEXT +); + +DROP TABLE IF EXISTS unicode_codepoint_script_mapping; + +CREATE TABLE unicode_codepoint_script_mapping ( + codepoint_from TEXT, + codepoint_to TEXT, + name TEXT +); + diff --git a/sources/languages/update.sh b/sources/languages/update.sh index 4604f7d..4441b88 100755 --- a/sources/languages/update.sh +++ b/sources/languages/update.sh @@ -13,6 +13,10 @@ REGISTRY_FILE="$DIR/language-subtag-registry" CLDR_URL="http://unicode.org/Public/cldr/latest/core.zip" CLDR_FILE="$DIR/cldr-core.zip" CLDR_DIR="$DIR/cldr" +UNICODE_SCRIPTS_URL="http://www.unicode.org/Public/UNIDATA/Scripts.txt" +UNICODE_SCRIPTS_FILE="$DIR/Scripts.txt" +PROPERTY_ALIASES_URL="http://www.unicode.org/Public/UNIDATA/PropertyValueAliases.txt" +PROPERTY_ALIASES_FILE="$DIR/PropertyValueAliases.txt" DATECMD='date +%Y-%m-%dT%H:%M:%S' @@ -36,7 +40,7 @@ sqlite3 $DATABASE <pre.sql echo "`$DATECMD` Getting subtag registry..." curl --silent --time-cond $REGISTRY_FILE --output $REGISTRY_FILE $REGISTRY_URL -echo "`$DATECMD` Running import..." +echo "`$DATECMD` Running subtag import..." ./import_subtag_registry.rb $DIR echo "`$DATECMD` Getting CLDR..." @@ -47,6 +51,13 @@ rm -fr $CLDR_DIR mkdir $CLDR_DIR unzip -q -d $CLDR_DIR $CLDR_FILE +echo "`$DATECMD` Getting unicode scripts..." +curl --silent --location --time-cond $UNICODE_SCRIPTS_FILE --output $UNICODE_SCRIPTS_FILE $UNICODE_SCRIPTS_URL +curl --silent --location --time-cond $PROPERTY_ALIASES_FILE --output $PROPERTY_ALIASES_FILE $PROPERTY_ALIASES_URL + +echo "`$DATECMD` Running unicode scripts import..." +./import_unicode_scripts.rb $DIR + echo "`$DATECMD` Running post.sql..." sqlite3 $DATABASE <post.sql diff --git a/taginfo-config-example.json b/taginfo-config-example.json index 04758e5..053426e 100644 --- a/taginfo-config-example.json +++ b/taginfo-config-example.json @@ -47,7 +47,7 @@ // Note that this will NOT work for the "db" source! Well, you can download it, // but it will fail later, because the database is changed by the master.sql // scripts. - "download": "josm potlatch merkaartor wiki", + "download": "languages josm potlatch merkaartor wiki", // These sources will be created from the actual sources. "create": "db", "db": { |