Integrate statistics handler, only us one db

Better integrate the statistics handler. It now uses the same database as the
tagstats handler (typically taginfo-db.db). The database is now given on the
command line. This simplifies the scripts calling tagstats.

6 files changed, 32 insertions, 52 deletions

diff --git a/sources/db/post.sql b/sources/db/post.sql
index c3486bb..bdd0fb7 100644
--- a/sources/db/post.sql
+++ b/sources/db/post.sql
@@ -12,10 +12,6 @@ PRAGMA count_changes = OFF;
 PRAGMA temp_store    = MEMORY;
 PRAGMA cache_size    = 5000000; 
 
-ATTACH DATABASE '__DIR__/count.db' AS count;
-INSERT INTO stats SELECT * FROM count.stats;
-DETACH DATABASE count;
-
 CREATE UNIQUE INDEX keys_key_idx ON keys (key);
 CREATE        INDEX tags_key_idx ON tags (key);
 -- CREATE UNIQUE INDEX tags_key_value_idx ON tags (key, value);
diff --git a/sources/db/update.sh b/sources/db/update.sh
index 4d10894..24e7f57 100755
--- a/sources/db/update.sh
+++ b/sources/db/update.sh
@@ -29,7 +29,6 @@ echo "`$DATECMD` Start db..."
 DATABASE=$DIR/taginfo-db.db
 
 rm -f $DATABASE
-rm -f $DIR/count.db
 
 echo "`$DATECMD` Running init.sql..."
 sqlite3 $DATABASE <../init.sql
@@ -46,17 +45,14 @@ left=`../../bin/taginfo-config.rb geodistribution.left`
 width=`../../bin/taginfo-config.rb geodistribution.width`
 height=`../../bin/taginfo-config.rb geodistribution.height`
 
-HERE=`pwd`
-cd $DIR
-#valgrind --leak-check=full --show-reachable=yes $HERE/tagstats $PLANETFILE >valgrind.log 2>&1
-$HERE/tagstats --left=$left --bottom=$bottom --top=$top --right=$right --width=$width --height=$height $PLANETFILE
-cd $HERE
+#valgrind --leak-check=full --show-reachable=yes $DIR/tagstats $PLANETFILE $DATABASE >valgrind.log 2>&1
+$DIR/tagstats --left=$left --bottom=$bottom --top=$top --right=$right --width=$width --height=$height $PLANETFILE $DATABASE
 
 echo "`$DATECMD` Running update_characters... "
 ./update_characters.pl $DIR
 
 echo "`$DATECMD` Running post.sql... "
-perl -pe "s|__DIR__|$DIR|" post.sql | sqlite3 $DATABASE
+sqlite3 $DATABASE <post.sql
 
 echo "`$DATECMD` Done db."
 
diff --git a/tagstats/statistics_handler.hpp b/tagstats/statistics_handler.hpp
index 7f5587e..dadbe7a 100644
--- a/tagstats/statistics_handler.hpp
+++ b/tagstats/statistics_handler.hpp
@@ -32,7 +32,7 @@ class StatisticsHandler : public Osmium::Handler::Base {
 
 public:
 
-    StatisticsHandler() : Base() {
+    StatisticsHandler(Osmium::Sqlite::Database& database) : Base(), m_database(database) {
         // if you change anything in this array, also change the corresponding struct below
         static const char *sn[] = {
             "nodes",
@@ -136,22 +136,8 @@ public:
     }
 
     void final() {
-        unlink("count.db");
-        Osmium::Sqlite::Database db("count.db");
-
-        sqlite3 *sqlite_db = db.get_sqlite3();
-        if (SQLITE_OK != sqlite3_exec(sqlite_db, \
-                                        "CREATE TABLE stats (" \
-                                        "  key    TEXT, " \
-                                        "  value  INT64 " \
-                                        ");", 0, 0, 0)) {
-            std::cerr << "Database error: " << sqlite3_errmsg(sqlite_db) << "\n";
-            sqlite3_close(sqlite_db);
-            exit(1);
-        }
-
-        Osmium::Sqlite::Statement *statement_insert_into_main_stats = db.prepare("INSERT INTO stats (key, value) VALUES (?, ?);");
-        db.begin_transaction();
+        Osmium::Sqlite::Statement* statement_insert_into_main_stats = m_database.prepare("INSERT INTO stats (key, value) VALUES (?, ?);");
+        m_database.begin_transaction();
 
         for (int i=0; m_stat_names[i]; ++i) {
             statement_insert_into_main_stats
@@ -164,7 +150,7 @@ public:
         ->bind_int64( ((uint64_t *) &m_stats)[0] - ((uint64_t *) &m_stats)[1] )
         ->execute();
 
-        db.commit();
+        m_database.commit();
 
         delete statement_insert_into_main_stats;
     }
@@ -204,6 +190,8 @@ private:
 
     const char **m_stat_names;
 
+    Osmium::Sqlite::Database& m_database;
+
     osm_object_id_t m_id;
     osm_version_t   m_version;
     int             m_tag_count;
diff --git a/tagstats/tagstats.cpp b/tagstats/tagstats.cpp
index 95394a7..5dcc4b9 100644
--- a/tagstats/tagstats.cpp
+++ b/tagstats/tagstats.cpp
@@ -51,7 +51,7 @@ int GeoDistribution::c_height;
 /* ================================================== */
 
 void print_help() {
-    std::cout << "tagstats [OPTIONS] OSMFILE\n\n" \
+    std::cout << "tagstats [OPTIONS] OSMFILE DATABASE\n\n" \
               << "This program is part of Taginfo. It calculates statistics\n" \
               << "on OSM tags and puts them into taginfo-db.db and count.db.\n" \
               << "\nOptions:\n" \
@@ -127,15 +127,16 @@ int main(int argc, char *argv[]) {
 
     Osmium::init(debug);
 
-    if (argc - optind != 1) {
-        std::cerr << "Usage: " << argv[0] << " [OPTIONS] OSMFILE" << std::endl;
+    if (argc - optind != 2) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] OSMFILE DATABASE" << std::endl;
         exit(1);
     }
 
     GeoDistribution::set_dimensions(width, height);
     Osmium::OSMFile infile(argv[optind]);
+    Osmium::Sqlite::Database db(argv[optind+1]);
     MapToInt<rough_position_t> map_to_int(left, bottom, right, top, width, height);
-    TagStatsHandler handler(map_to_int);
+    TagStatsHandler handler(db, map_to_int);
     infile.read(handler);
 }
 
diff --git a/tagstats/tagstats_handler.hpp b/tagstats/tagstats_handler.hpp
index 2e66853..ee419a1 100644
--- a/tagstats/tagstats_handler.hpp
+++ b/tagstats/tagstats_handler.hpp
@@ -174,7 +174,7 @@ class TagStatsHandler : public Osmium::Handler::Base {
     static const int string_store_size = 1024 * 1024 * 10;
     StringStore* string_store;
 
-    Osmium::Sqlite::Database* db;
+    Osmium::Sqlite::Database& m_database;
 
     void _timer_info(const char *msg) {
         int duration = time(0) - timer;
@@ -205,8 +205,8 @@ class TagStatsHandler : public Osmium::Handler::Base {
     void _print_and_clear_distribution_images(bool for_nodes) {
         int sum_size=0;
 
-        Osmium::Sqlite::Statement* statement_insert_into_key_distributions = db->prepare("INSERT INTO key_distributions (key, object_type, png) VALUES (?, ?, ?);");
-        db->begin_transaction();
+        Osmium::Sqlite::Statement* statement_insert_into_key_distributions = m_database.prepare("INSERT INTO key_distributions (key, object_type, png) VALUES (?, ?, ?);");
+        m_database.begin_transaction();
 
         for (key_hash_map_t::const_iterator it = tags_stat.begin(); it != tags_stat.end(); it++) {
             KeyStats* stat = it->second;
@@ -235,7 +235,7 @@ class TagStatsHandler : public Osmium::Handler::Base {
         std::cerr << "gridcells_all: " << GeoDistribution::count_all_set_cells() << std::endl;
         std::cerr << "sum of location image sizes: " << sum_size << std::endl;
 
-        db->commit();
+        m_database.commit();
         delete statement_insert_into_key_distributions;
     }
 
@@ -314,21 +314,20 @@ class TagStatsHandler : public Osmium::Handler::Base {
 
 public:
 
-    TagStatsHandler(MapToInt<rough_position_t>& map_to_int) :
+    TagStatsHandler(Osmium::Sqlite::Database& database, MapToInt<rough_position_t>& map_to_int) :
         Base(),
         max_timestamp(0),
-        statistics_handler(),
+        m_database(database),
+        statistics_handler(database),
         m_map_to_int(map_to_int)
 #ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
         , m_storage()
 #endif
     {
         string_store = new StringStore(string_store_size);
-        db = new Osmium::Sqlite::Database("taginfo-db.db");
     }
 
     ~TagStatsHandler() {
-        delete db;
         delete string_store;
     }
 
@@ -364,12 +363,12 @@ public:
 
         int size;
         void* ptr = GeoDistribution::create_empty_png(&size);
-        Osmium::Sqlite::Statement* statement_insert_into_key_distributions = db->prepare("INSERT INTO key_distributions (png) VALUES (?);");
-        db->begin_transaction();
+        Osmium::Sqlite::Statement* statement_insert_into_key_distributions = m_database.prepare("INSERT INTO key_distributions (png) VALUES (?);");
+        m_database.begin_transaction();
         statement_insert_into_key_distributions
         ->bind_blob(ptr, size) // column: png
         ->execute();
-        db->commit();
+        m_database.commit();
         delete statement_insert_into_key_distributions;
 
         _print_and_clear_distribution_images(true);
@@ -422,25 +421,25 @@ public:
         _print_memory_usage();
         timer = time(0);
 
-        Osmium::Sqlite::Statement *statement_insert_into_keys = db->prepare("INSERT INTO keys (key, " \
+        Osmium::Sqlite::Statement *statement_insert_into_keys = m_database.prepare("INSERT INTO keys (key, " \
                 " count_all,  count_nodes,  count_ways,  count_relations, " \
                 "values_all, values_nodes, values_ways, values_relations, " \
                 " users_all, " \
                 "cells_nodes, cells_ways) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
 
-        Osmium::Sqlite::Statement *statement_insert_into_tags = db->prepare("INSERT INTO tags (key, value, " \
+        Osmium::Sqlite::Statement *statement_insert_into_tags = m_database.prepare("INSERT INTO tags (key, value, " \
                 "count_all, count_nodes, count_ways, count_relations) " \
                 "VALUES (?, ?, ?, ?, ?, ?);");
 
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
-        Osmium::Sqlite::Statement* statement_insert_into_key_combinations = db->prepare("INSERT INTO keypairs (key1, key2, " \
+        Osmium::Sqlite::Statement* statement_insert_into_key_combinations = m_database.prepare("INSERT INTO keypairs (key1, key2, " \
                 "count_all, count_nodes, count_ways, count_relations) " \
                 "VALUES (?, ?, ?, ?, ?, ?);");
 #endif // TAGSTATS_COUNT_KEY_COMBINATIONS
 
-        Osmium::Sqlite::Statement* statement_update_meta = db->prepare("UPDATE source SET data_until=?");
+        Osmium::Sqlite::Statement* statement_update_meta = m_database.prepare("UPDATE source SET data_until=?");
 
-        db->begin_transaction();
+        m_database.begin_transaction();
 
         struct tm* tm = gmtime(&max_timestamp);
         static char max_timestamp_str[20]; // thats enough space for the timestamp generated from the pattern in the next line
@@ -524,7 +523,7 @@ public:
             delete stat; // lets make valgrind happy
         }
 
-        db->commit();
+        m_database.commit();
 
         delete statement_update_meta;
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
diff --git a/tagstats/test_tagstats.sh b/tagstats/test_tagstats.sh
index 0146754..da05e97 100755
--- a/tagstats/test_tagstats.sh
+++ b/tagstats/test_tagstats.sh
@@ -20,6 +20,6 @@ sqlite3 $DATABASE <../sources/db/pre.sql
 ulimit -c 1000000000
 rm -f core
 
-#./tagstats --left=5.5 --bottom=47 --right=15 --top=55 --width=200 --height=320 $OSMFILE
-./tagstats $OSMFILE
+#./tagstats --left=5.5 --bottom=47 --right=15 --top=55 --width=200 --height=320 $OSMFILE $DATABASE
+./tagstats $OSMFILE $DATABASE