Extend languages source to get info about scripts.

4 files changed, 100 insertions, 2 deletions

diff --git a/sources/languages/import_unicode_scripts.rb b/sources/languages/import_unicode_scripts.rb
new file mode 100755
index 0000000..7855a78
--- /dev/null
+++ b/sources/languages/import_unicode_scripts.rb
@@ -0,0 +1,72 @@
+#!/usr/bin/ruby
+#------------------------------------------------------------------------------
+#
+#  Taginfo source: Languages
+#
+#  import_unicode_scripts.rb
+#
+#------------------------------------------------------------------------------
+#
+#  Copyright (C) 2013  Jochen Topf <jochen@remote.org>
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+#------------------------------------------------------------------------------
+
+require 'rubygems'
+
+require 'sqlite3'
+
+dir = ARGV[0] || '.'
+
+db = SQLite3::Database.new(dir + '/taginfo-languages.db')
+
+property_value_alias_file = "#{dir}/PropertyValueAliases.txt"
+codepoint_script_mapping_file = "#{dir}/Scripts.txt"
+
+db.execute('BEGIN TRANSACTION');
+
+open(property_value_alias_file) do |file|
+    file.each do |line|
+        line.chomp!
+        if line.match(%r{^sc ;})
+            (sc, script, name) = line.split(%r{\s*;\s*})
+            db.execute("INSERT INTO unicode_scripts (script, name) VALUES (?, ?)", script, name)
+        end
+    end
+end
+
+open(codepoint_script_mapping_file) do |file|
+    file.each do |line|
+        line.chomp!
+        line.sub!(%r{\s*#.*}, '')
+        next if line.match(%r{^$})
+        (codes, script) = line.split(%r{\s+;\s+})
+        if codes.match(%r{^[0-9A-F]{4,5}$})
+            from = codes
+            to   = codes
+        elsif codes.match(%r{^([0-9A-F]{4,5})..([0-9A-F]{4,5})$})
+            from = $1
+            to   = $2
+        else
+            puts "Line does not match: #{line}"
+            next
+        end
+        db.execute("INSERT INTO unicode_codepoint_script_mapping (codepoint_from, codepoint_to, name) VALUES (?, ?, ?)", from, to, script)
+    end
+end
+
+db.execute('COMMIT');
+
diff --git a/sources/languages/pre.sql b/sources/languages/pre.sql
index c1a505c..2cf0aa7 100644
--- a/sources/languages/pre.sql
+++ b/sources/languages/pre.sql
@@ -20,3 +20,18 @@ CREATE TABLE subtags (
     prefix          TEXT
 );
 
+DROP TABLE IF EXISTS unicode_scripts;
+
+CREATE TABLE unicode_scripts (
+    script TEXT,
+    name   TEXT
+);
+
+DROP TABLE IF EXISTS unicode_codepoint_script_mapping;
+
+CREATE TABLE unicode_codepoint_script_mapping (
+    codepoint_from TEXT,
+    codepoint_to   TEXT,
+    name           TEXT
+);
+
diff --git a/sources/languages/update.sh b/sources/languages/update.sh
index 4604f7d..4441b88 100755
--- a/sources/languages/update.sh
+++ b/sources/languages/update.sh
@@ -13,6 +13,10 @@ REGISTRY_FILE="$DIR/language-subtag-registry"
 CLDR_URL="http://unicode.org/Public/cldr/latest/core.zip"
 CLDR_FILE="$DIR/cldr-core.zip"
 CLDR_DIR="$DIR/cldr"
+UNICODE_SCRIPTS_URL="http://www.unicode.org/Public/UNIDATA/Scripts.txt"
+UNICODE_SCRIPTS_FILE="$DIR/Scripts.txt"
+PROPERTY_ALIASES_URL="http://www.unicode.org/Public/UNIDATA/PropertyValueAliases.txt"
+PROPERTY_ALIASES_FILE="$DIR/PropertyValueAliases.txt"
 
 DATECMD='date +%Y-%m-%dT%H:%M:%S'
 
@@ -36,7 +40,7 @@ sqlite3 $DATABASE <pre.sql
 echo "`$DATECMD` Getting subtag registry..."
 curl --silent --time-cond $REGISTRY_FILE --output $REGISTRY_FILE $REGISTRY_URL
 
-echo "`$DATECMD` Running import..."
+echo "`$DATECMD` Running subtag import..."
 ./import_subtag_registry.rb $DIR
 
 echo "`$DATECMD` Getting CLDR..."
@@ -47,6 +51,13 @@ rm -fr $CLDR_DIR
 mkdir $CLDR_DIR
 unzip -q -d $CLDR_DIR $CLDR_FILE
 
+echo "`$DATECMD` Getting unicode scripts..."
+curl --silent --location --time-cond $UNICODE_SCRIPTS_FILE --output $UNICODE_SCRIPTS_FILE $UNICODE_SCRIPTS_URL
+curl --silent --location --time-cond $PROPERTY_ALIASES_FILE --output $PROPERTY_ALIASES_FILE $PROPERTY_ALIASES_URL
+
+echo "`$DATECMD` Running unicode scripts import..."
+./import_unicode_scripts.rb $DIR
+
 echo "`$DATECMD` Running post.sql..."
 sqlite3 $DATABASE <post.sql
 
diff --git a/taginfo-config-example.json b/taginfo-config-example.json
index 04758e5..053426e 100644
--- a/taginfo-config-example.json
+++ b/taginfo-config-example.json
@@ -47,7 +47,7 @@
         // Note that this will NOT work for the "db" source! Well, you can download it,
         // but it will fail later, because the database is changed by the master.sql
         // scripts.
-        "download": "josm potlatch merkaartor wiki",
+        "download": "languages josm potlatch merkaartor wiki",
         // These sources will be created from the actual sources.
         "create": "db",
         "db": {