Generate distribution maps for keys on way, too

Until now distribution maps were only generated for keys on nodes. No there are
maps for keys on ways, too. (Not for relations, though). This commit also adds
command line options for tagstats to define the area you want the map for. Its
no longer necessary to recompile tagstats if you want to change this.
This commit does not change the UI/API yet, so you still only get the node map.

8 files changed, 352 insertions, 164 deletions

diff --git a/examples/get_distribution_image.rb b/examples/get_distribution_image.rb
index 96500a4..b840a1a 100755
--- a/examples/get_distribution_image.rb
+++ b/examples/get_distribution_image.rb
@@ -1,17 +1,18 @@
 #!/usr/bin/ruby
 #
-#  get_distribution_image DB KEY
+#  get_distribution_image DB KEY nodes|ways
 #
 
 require 'rubygems'
 require 'sqlite3'
 
 filename = ARGV[0]
-key = ARGV[1]
+key      = ARGV[1]
+type     = ARGV[2]
 
 db = SQLite3::Database.new(filename)
 
-db.execute("SELECT png FROM key_distributions WHERE key=?", key) do |row|
-    puts row[0]
+db.execute("SELECT #{type} FROM key_distributions WHERE key=?", key) do |row|
+    $stdout.write row[0]
 end
 
diff --git a/sources/db/pre.sql b/sources/db/pre.sql
index b91e31e..64e28bd 100644
--- a/sources/db/pre.sql
+++ b/sources/db/pre.sql
@@ -24,7 +24,8 @@ CREATE TABLE keys (
   users_nodes      INTEGER DEFAULT 0,
   users_ways       INTEGER DEFAULT 0,
   users_relations  INTEGER DEFAULT 0,
-  grids            INTEGER DEFAULT 0,
+  cells_nodes      INTEGER DEFAULT 0,
+  cells_ways       INTEGER DEFAULT 0,
   in_wiki          INTEGER DEFAULT 0,
   in_wiki_en       INTEGER DEFAULT 0,
   in_josm          INTEGER DEFAULT 0,
@@ -47,7 +48,8 @@ DROP TABLE IF EXISTS key_distributions;
 
 CREATE TABLE key_distributions (
   key              VARCHAR,
-  png              BLOB
+  nodes            BLOB,
+  ways             BLOB
 );
 
 DROP TABLE IF EXISTS tags;
diff --git a/tagstats/Makefile b/tagstats/Makefile
index bb28d29..1543335 100644
--- a/tagstats/Makefile
+++ b/tagstats/Makefile
@@ -15,6 +15,10 @@ CXXFLAGS += -std=c++0x -Wall -Wextra -Wredundant-decls -Wdisabled-optimization -
 CXXFLAGS += -DTAGSTATS_COUNT_USERS
 CXXFLAGS += -DTAGSTATS_COUNT_KEY_COMBINATIONS
 
+CXXFLAGS += -DTAGSTATS_GEODISTRIBUTION_INT=uint16_t
+CXXFLAGS += -DTAGSTATS_GEODISTRIBUTION_FOR_WAYS=SparseTable
+#CXXFLAGS += -DTAGSTATS_GEODISTRIBUTION_FOR_WAYS=Mmap
+
 CXXFLAGS += -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64
 
 CXXFLAGS += -I../../../osmium/osmium/include
diff --git a/tagstats/geodistribution.hpp b/tagstats/geodistribution.hpp
index 30a1303..ce5ece4 100644
--- a/tagstats/geodistribution.hpp
+++ b/tagstats/geodistribution.hpp
@@ -1,16 +1,84 @@
 #ifndef TAGSTATS_GEODISTRIBUTION_HPP
 #define TAGSTATS_GEODISTRIBUTION_HPP
 
-#include <bitset>
+#include <stdexcept>
+#include <limits>
 
 #include <gd.h>
 
-class GeoDistribution {
+/**
+ * Functor class defining the () operator as a function that limits a
+ * Osmium::OSM::Position to a bounding box, reduces the resolution
+ * of the coordinates and returns an integer.
+ * 
+ * @tparam T Result type after conversion. Must be an unsigned integer type.
+ */
+template <typename T>
+class MapToInt {
+
+    double m_minx;
+    double m_miny;
+    double m_maxx;
+    double m_maxy;
+
+    unsigned int m_width;
+    unsigned int m_height;
+
+    double m_dx;
+    double m_dy;
+
+public:
+
+    MapToInt(double minx = -180, double miny = -90, double maxx = 180, double maxy = 90, unsigned int width = 360, unsigned int height = 180) :
+        m_minx(minx), m_miny(miny), m_maxx(maxx), m_maxy(maxy),
+        m_width(width), m_height(height),
+        m_dx(maxx - minx), m_dy(maxy - miny) {
+        if (size() > std::numeric_limits<T>::max()) {
+            throw std::range_error("width*height must be smaller than MAXINT for type T");
+        }
+    }
+
+    T operator()(const Osmium::OSM::Position& p) const {
+        if (p.lon() < m_minx || p.lat() < m_miny || p.lon() >= m_maxx || p.lat() >= m_maxy) {
+            throw std::range_error("position out of bounds");
+        }
+        int x = (p.lon() - m_minx) / m_dx * m_width;
+        int y = (m_maxy - p.lat()) / m_dy * m_height;
+
+        if (x < 0) {
+            x = 0;
+        } else if (static_cast<unsigned int>(x) >= m_width) {
+            x = m_width-1;
+        }
+        if (y < 0) {
+            y = 0;
+        } else if (static_cast<unsigned int>(y) >= m_height) {
+            y = m_height-1;
+        }
+
+        return y * m_width + x;
+    }
+
+    unsigned int width() const {
+        return m_width;
+    }
+
+    unsigned int height() const {
+        return m_height;
+    }
+
+    unsigned int size() const {
+        return m_width * m_height;
+    }
+
+};
 
-    static const int resolution_y = 360;
-    static const int resolution_x = 2 * resolution_y;
+/**
+ * Stores the geographical distribution of something in a space efficient way.
+ */
+class GeoDistribution {
 
-    typedef std::bitset<resolution_x * resolution_y> geo_distribution_t;
+    typedef std::vector<bool> geo_distribution_t;
 
     /**
      * Contains a pointer to a bitset that gives us the distribution.
@@ -18,59 +86,65 @@ class GeoDistribution {
      * if more than one grid cell is used, we dynamically create an
      * object for this.
      */
-    geo_distribution_t *distribution;
+    geo_distribution_t* m_distribution;
 
     /**
-     * Number of grid cells.
-     * Will be 0 in the beginning, 1 if there is only one grid cell and
-     * 2 if there are two or more.
+     * Number of set grid cells.
      */
-    int cells;
+    unsigned int m_cells;
 
-    /// If there is only one grid cell location, this is where its kept
-    int location;
+    /// If there is only one grid cell location, this is where its kept.
+    rough_position_t m_location;
 
     /// Overall distribution
-    static geo_distribution_t distribution_all;
+    static geo_distribution_t c_distribution_all;
+
+    static int c_width;
+    static int c_height;
 
 public:
 
-    GeoDistribution() : distribution(NULL), cells(0), location(-1) {
+    GeoDistribution() : m_distribution(NULL), m_cells(0), m_location(0) {
     }
 
     ~GeoDistribution() {
-        delete distribution;
+        delete m_distribution;
+    }
+
+    void clear() {
+        delete m_distribution;
+        m_distribution = NULL;
+        m_cells = 0;
+        m_location = 0;
+    }
+
+    static void set_dimensions(int width, int height) {
+        c_width = width;
+        c_height = height;
+        c_distribution_all.resize(c_width * c_height);
     }
 
     /**
      * Add the given coordinate to the distribution store.
      */
-    void add_coordinate(double dx, double dy) {
-        int x =                int(2 * (dx + 180));
-        int y = resolution_y - int(2 * (dy +  90));
-
-        if (x < 0) x=0;
-        if (y < 0) y=0;
-        if (x >= resolution_x) x = resolution_x-1;
-        if (y >= resolution_y) y = resolution_y-1;
-
-        int n = resolution_x * y + x;
-        if (cells == 0) {
-            location = n;
-            cells++;
-            distribution_all[n] = true;
-        } else if (cells == 1) {
-            if (location != n) {
-                distribution = new geo_distribution_t;
-                (*distribution)[location] = true;
-                distribution_all[location] = true;
-                (*distribution)[n] = true;
-                distribution_all[n] = true;
-                cells++;
-            }
-        } else {
-            (*distribution)[n] = true;
-            distribution_all[n] = true;
+    void add_coordinate(rough_position_t n) {
+        if (m_cells == 0) {
+            m_location = n;
+            m_cells++;
+            c_distribution_all[n] = true;
+        } else if (m_cells == 1 && m_location != n) {
+            m_distribution = new geo_distribution_t(c_width*c_height);
+            (*m_distribution)[m_location] = true;
+            c_distribution_all[m_location] = true;
+            (*m_distribution)[n] = true;
+            c_distribution_all[n] = true;
+            m_cells++;
+        } else if (m_cells == 1 && m_location == n) {
+            // nothing to do
+        } else if (! (*m_distribution)[n]) {
+            m_cells++;
+            (*m_distribution)[n] = true;
+            c_distribution_all[n] = true;
         }
     }
 
@@ -83,27 +157,26 @@ public:
      *        image.
      * @returns Pointer to memory area with PNG image.
      */
-    void *create_png(int *size) {
-        gdImagePtr im = gdImageCreate(resolution_x, resolution_y);
+    void* create_png(int* size) {
+        gdImagePtr im = gdImageCreate(c_width, c_height);
         int bgColor = gdImageColorAllocate(im, 0, 0, 0);
         gdImageColorTransparent(im, bgColor);
         int fgColor = gdImageColorAllocate(im, 180, 0, 0);
 
-        if (distribution) {
+        if (m_cells == 1) {
+            int y = m_location / c_width;
+            int x = m_location - (y * c_width);
+            gdImageSetPixel(im, x, y, fgColor);
+        } else if (m_cells >= 2) {
             int n=0;
-            for (int y=0; y < resolution_y; y++) {
-                for (int x=0; x < resolution_x; x++) {
-                    if ((*distribution)[n]) {
-                        cells++;
+            for (int y=0; y < c_height; y++) {
+                for (int x=0; x < c_width; x++) {
+                    if ((*m_distribution)[n]) {
                         gdImageSetPixel(im, x, y, fgColor);
                     }
                     n++;
                 }
             }
-        } else {
-            int y = location / resolution_x;
-            int x = location - y;
-            gdImageSetPixel(im, x, y, fgColor);
         }
 
         void *png = gdImagePngPtr(im, size);
@@ -120,26 +193,26 @@ public:
     }
 
     /**
-     * Return the number of cells set. This is only valid after a call to create_png().
+     * Return the number of cells set.
      */
-    int get_cells() const {
-        return cells;
+    unsigned int cells() const {
+        return m_cells;
     }
 
     /**
      * Return the number of cells that are set in at least one GeoDistribution
      * object.
      */
-    static int count_all_set_cells() {
-        return distribution_all.count();
+    static unsigned int count_all_set_cells() {
+        int c=0;
+        for (int n=0; n < c_width*c_height; ++n) {
+            if (c_distribution_all[n]) {
+                c++;
+            }
+        }
+        return c;
     }
 
-    /**
-     * Resets the distribution storage for the overall distribution.
-     */
-    static void reset() {
-        distribution_all.reset();
-    }
 };
 
 #endif // TAGSTATS_GEODISTRIBUTION_HPP
diff --git a/tagstats/tagstats.cpp b/tagstats/tagstats.cpp
index fc2eb57..4b27b9c 100644
--- a/tagstats/tagstats.cpp
+++ b/tagstats/tagstats.cpp
@@ -1,87 +1,118 @@
 
+#include <getopt.h>
+
 #include <osmium.hpp>
 
 #include <osmium/handler/statistics.hpp>
-//#include <osmium/handler/node_location_store.hpp>
-#include "geodistribution.hpp"
-#include "tagstats_handler.hpp"
-
-GeoDistribution::geo_distribution_t GeoDistribution::distribution_all;
-
-class MyTagStatsHandler : public Osmium::Handler::Base {
-
-    Osmium::Handler::Statistics      osmium_handler_stats;
-    TagStatsHandler                  osmium_handler_tagstats;
-    //Osmium::Handler::NLS_Sparsetable osmium_handler_node_location_store;
-
-public:
 
-    void init(Osmium::OSM::Meta& meta) {
-        osmium_handler_tagstats.init(meta);
-        // osmium_handler_node_location_store.init(meta);
-    }
-
-    void before_nodes() {
-        osmium_handler_tagstats.before_nodes();
-    }
+/**
+ * Positions are stored in this type of integer for the distribution images.
+ * TAGSTATS_GEODISTRIBUTION_INT must be set in Makefile, typically to uint16_t
+ * or uint32_t (for higher resolution but needs twice as much memory).
+ */
+typedef TAGSTATS_GEODISTRIBUTION_INT rough_position_t;
 
-    void node(Osmium::OSM::Node *node) {
-        osmium_handler_stats.node(node);
-        osmium_handler_tagstats.node(node);
-        //    osmium_handler_node_location_store.node(node);
-    }
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+# include <osmium/storage/byid.hpp>
+// Set TAGSTATS_GEODISTRIBUTION_STORAGE to SparseTable or Mmap in Makefile
+typedef Osmium::Storage::TAGSTATS_GEODISTRIBUTION_FOR_WAYS<rough_position_t> storage_t;
+#endif // TAGSTATS_GEODISTRIBUTION_FOR_WAYS
 
-    void after_nodes() {
-        osmium_handler_tagstats.after_nodes();
-    }
-
-    void before_ways() {
-        osmium_handler_tagstats.before_ways();
-    }
+#include "geodistribution.hpp"
 
-    void way(Osmium::OSM::Way *way) {
-        osmium_handler_stats.way(way);
-        osmium_handler_tagstats.way(way);
-        //    osmium_handler_node_location_store.way(way);
-    }
+GeoDistribution::geo_distribution_t GeoDistribution::c_distribution_all;
+int GeoDistribution::c_width;
+int GeoDistribution::c_height;
 
-    void after_ways() {
-        osmium_handler_tagstats.after_ways();
-    }
+#include "tagstats_handler.hpp"
 
-    void before_relations() {
-        osmium_handler_tagstats.before_relations();
-    }
 
-    void relation(Osmium::OSM::Relation *relation) {
-        osmium_handler_stats.relation(relation);
-        osmium_handler_tagstats.relation(relation);
-    }
+/* ================================================== */
 
-    void after_relations() {
-        osmium_handler_tagstats.after_relations();
-    }
+void print_help() {
+    std::cout << "tagstats [OPTIONS] OSMFILE\n\n" \
+              << "This program is part of Taginfo. It calculates statistics\n" \
+              << "on OSM tags and puts them into taginfo-db.db and count.db.\n" \
+              << "\nOptions:\n" \
+              << "  -H, --help           This help message\n" \
+              << "  -d, --debug          Enable debugging output\n" \
+              << "  -t, --top=NUMBER     Top of bounding box for distribution images\n" \
+              << "  -r, --right=NUMBER   Right of bounding box for distribution images\n" \
+              << "  -b, --bottom=NUMBER  Bottom of bounding box for distribution images\n" \
+              << "  -l, --left=NUMBER    Left of bounding box for distribution images\n" \
+              << "  -w, --width=NUMBER   Width of distribution images (default: 360)\n" \
+              << "  -h, --height=NUMBER  Height of distribution images (default: 180)\n" \
+              << "\nDefault for bounding box is: (-180, -90, 180, 90)\n";
+}
 
-    void final() {
-        // osmium_handler_node_location_store.final();
-        osmium_handler_stats.final();
-        osmium_handler_tagstats.final();
+int main(int argc, char *argv[]) {
+    static struct option long_options[] = {
+        {"debug",  no_argument, 0, 'd'},
+        {"help",   no_argument, 0, 'H'},
+        {"top",    required_argument, 0, 't'},
+        {"right",  required_argument, 0, 'r'},
+        {"bottom", required_argument, 0, 'b'},
+        {"left",   required_argument, 0, 'l'},
+        {"width",  required_argument, 0, 'w'},
+        {"height", required_argument, 0, 'h'},
+        {0, 0, 0, 0}
+    };
+
+    bool debug = false;
+
+    int top    =   90;
+    int right  =  180;
+    int bottom =  -90;
+    int left   = -180;
+
+    unsigned int width  = 360;
+    unsigned int height = 180;
+
+    while (true) {
+        int c = getopt_long(argc, argv, "dHt:r:b:l:w:h:", long_options, 0);
+        if (c == -1) {
+            break;
+        }
+
+        switch (c) {
+            case 'd':
+                debug = true;
+                break;
+            case 'H':
+                print_help();
+                exit(0);
+            case 't':
+                top = atoi(optarg);
+                break;
+            case 'r':
+                right = atoi(optarg);
+                break;
+            case 'b':
+                bottom = atoi(optarg);
+                break;
+            case 'l':
+                left = atoi(optarg);
+                break;
+            case 'w':
+                width = atoi(optarg);
+                break;
+            case 'h':
+                height = atoi(optarg);
+                break;
+            default:
+                exit(1);
+        }
     }
-};
 
-/* ================================================== */
-
-int main(int argc, char *argv[]) {
-    Osmium::init();
+    Osmium::init(debug);
 
-    GeoDistribution::reset();
-    if (argc != 2) {
-        std::cerr << "Usage: " << argv[0] << " OSMFILE" << std::endl;
+    if (argc - optind != 1) {
+        std::cerr << "Usage: " << argv[0] << " [OPTIONS] OSMFILE" << std::endl;
         exit(1);
     }
 
-    Osmium::OSMFile infile(argv[1]);
-    MyTagStatsHandler handler;
+    Osmium::OSMFile infile(argv[optind]);
+    TagStatsHandler handler(left, bottom, right, top, width, height);
     infile.read(handler);
 }
 
diff --git a/tagstats/tagstats_handler.hpp b/tagstats/tagstats_handler.hpp
index 144c2ea..0e791bb 100644
--- a/tagstats/tagstats_handler.hpp
+++ b/tagstats/tagstats_handler.hpp
@@ -79,6 +79,7 @@ public:
 
     Counter key;
     Counter values;
+    Counter cells;
 
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
     key_combination_hash_map_t key_combination_hash;
@@ -90,28 +91,30 @@ public:
 
     value_hash_map_t values_hash;
 
+    GeoDistribution distribution;
+
     KeyStats()
         : key(),
           values(),
+          cells(),
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
           key_combination_hash(),
 #endif // TAGSTATS_COUNT_KEY_COMBINATIONS
 #ifdef TAGSTATS_COUNT_USERS
           user_hash(),
 #endif // TAGSTATS_COUNT_USERS
-          values_hash() {
+          values_hash(),
+          distribution() {
     }
 
-    GeoDistribution node_distribution;
-
-    void update(const char *value, Osmium::OSM::Object *object, StringStore *string_store) {
+    void update(const char* value, Osmium::OSM::Object* object, StringStore* string_store) {
         key.count[object->get_type()]++;
 
         value_hash_map_t::iterator values_iterator(values_hash.find(value));
         if (values_iterator == values_hash.end()) {
             Counter counter;
             counter.count[object->get_type()] = 1;
-            values_hash.insert(std::pair<const char *, Counter>(string_store->add(value), counter));
+            values_hash.insert(std::pair<const char*, Counter>(string_store->add(value), counter));
             values.count[object->get_type()]++;
         } else {
             values_iterator->second.count[object->get_type()]++;
@@ -123,11 +126,6 @@ public:
 #ifdef TAGSTATS_COUNT_USERS
         user_hash[object->uid()]++;
 #endif // TAGSTATS_COUNT_USERS
-
-        if (object->get_type() == NODE) {
-            node_distribution.add_coordinate(static_cast<Osmium::OSM::Node *>(object)->get_lon(),
-                                             static_cast<Osmium::OSM::Node *>(object)->get_lat());
-        }
     }
 
     void add_key_kombination(const char *other_key, osm_object_id_t type) {
@@ -152,9 +150,9 @@ class TagStatsHandler : public Osmium::Handler::Base {
 
     // this must be much bigger than the largest string we want to store
     static const int string_store_size = 1024 * 1024 * 10;
-    StringStore *string_store;
+    StringStore* string_store;
 
-    Osmium::Sqlite::Database *db;
+    Osmium::Sqlite::Database* db;
 
     void _timer_info(const char *msg) {
         int duration = time(0) - timer;
@@ -182,24 +180,33 @@ class TagStatsHandler : public Osmium::Handler::Base {
     }
 #endif // TAGSTATS_COUNT_KEY_COMBINATIONS
 
-    void _print_images() {
+    void _print_and_clear_distribution_images(bool for_nodes) {
         int sum_size=0;
 
-        Osmium::Sqlite::Statement *statement_insert_into_key_distributions = db->prepare("INSERT INTO key_distributions (key, png) VALUES (?, ?);");
+        Osmium::Sqlite::Statement* statement_insert_into_key_distributions = db->prepare(for_nodes ? "INSERT INTO key_distributions (nodes, key) VALUES (?, ?);" : "UPDATE key_distributions SET ways=? WHERE key=?");
         db->begin_transaction();
 
-        for (key_hash_map_t::const_iterator it(tags_stat.begin()); it != tags_stat.end(); it++) {
-            KeyStats *stat = it->second;
+        for (key_hash_map_t::const_iterator it = tags_stat.begin(); it != tags_stat.end(); it++) {
+            KeyStats* stat = it->second;
+
+            if (for_nodes) {
+                stat->cells.count[NODE] = stat->distribution.cells();
+            } else {
+                stat->cells.count[WAY] = stat->distribution.cells();
+            }
 
             int size;
-            void *ptr = stat->node_distribution.create_png(&size);
+            void *ptr = stat->distribution.create_png(&size);
             sum_size += size;
+
             statement_insert_into_key_distributions
+            ->bind_blob(ptr, size) // column: nodes/ways
             ->bind_text(it->first) // column: key
-            ->bind_blob(ptr, size) // column: png
             ->execute();
 
-            stat->node_distribution.free_png(ptr);
+            stat->distribution.free_png(ptr);
+
+            stat->distribution.clear();
         }
 
         std::cerr << "gridcells_all: " << GeoDistribution::count_all_set_cells() << std::endl;
@@ -251,6 +258,22 @@ class TagStatsHandler : public Osmium::Handler::Base {
                 stat = tags_iterator->second;
             }
             stat->update(it->value(), object, string_store);
+
+            if (object->get_type() == NODE) {
+                try {
+                    stat->distribution.add_coordinate(m_map_to_int(static_cast<Osmium::OSM::Node*>(object)->position()));
+                } catch (std::range_error) {
+                    // ignore
+                }
+            }
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+            else if (object->get_type() == WAY) {
+                // This will only add the coordinate of the first node in a way to the
+                // distribution. We'll see how this goes, maybe we need to store the
+                // coordinates of all nodes?
+                stat->distribution.add_coordinate(m_storage[static_cast<Osmium::OSM::Way*>(object)->nodes().front().ref()]);
+            }
+#endif // TAGSTATS_GEODISTRIBUTION_FOR_WAYS
         }
 
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
@@ -258,11 +281,28 @@ class TagStatsHandler : public Osmium::Handler::Base {
 #endif // TAGSTATS_COUNT_KEY_COMBINATIONS
     }
 
+    Osmium::Handler::Statistics osmium_handler_stats;
+
+    MapToInt<rough_position_t> m_map_to_int;
+
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+    storage_t m_storage;
+#endif
+
 public:
 
-    TagStatsHandler() : Base(), max_timestamp(0) {
+    TagStatsHandler(double minx, double miny, double maxx, double maxy, unsigned int width, unsigned int height) :
+        Base(),
+        max_timestamp(0),
+        osmium_handler_stats(),
+        m_map_to_int(minx, miny, maxx, maxy, width, height)
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+        , m_storage()
+#endif
+        {
         string_store = new StringStore(string_store_size);
         db = new Osmium::Sqlite::Database("taginfo-db.db");
+        GeoDistribution::set_dimensions(width, height);
     }
 
     ~TagStatsHandler() {
@@ -270,15 +310,25 @@ public:
         delete string_store;
     }
 
-    void node(Osmium::OSM::Node *node) {
+    void node(Osmium::OSM::Node* node) {
+        osmium_handler_stats.node(node);
         collect_tag_stats(node);
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+        try {
+            m_storage.set(node->id(), m_map_to_int(node->position()));
+        } catch (std::range_error) {
+            // ignore
+        }
+#endif
     }
 
-    void way(Osmium::OSM::Way *way) {
+    void way(Osmium::OSM::Way* way) {
+        osmium_handler_stats.way(way);
         collect_tag_stats(way);
     }
 
-    void relation(Osmium::OSM::Relation *relation) {
+    void relation(Osmium::OSM::Relation* relation) {
+        osmium_handler_stats.relation(relation);
         collect_tag_stats(relation);
     }
 
@@ -289,8 +339,8 @@ public:
     void after_nodes() {
         _timer_info("processing nodes");
         _print_memory_usage();
+        _print_and_clear_distribution_images(true);
         timer = time(0);
-        _print_images();
         _timer_info("dumping images");
         _print_memory_usage();
     }
@@ -301,6 +351,9 @@ public:
 
     void after_ways() {
         _timer_info("processing ways");
+#ifdef TAGSTATS_GEODISTRIBUTION_FOR_WAYS
+        _print_and_clear_distribution_images(false);
+#endif
         _print_memory_usage();
     }
 
@@ -332,6 +385,7 @@ public:
     }
 
     void final() {
+        osmium_handler_stats.final();
         _print_memory_usage();
         timer = time(0);
 
@@ -339,23 +393,23 @@ public:
                 " count_all,  count_nodes,  count_ways,  count_relations, " \
                 "values_all, values_nodes, values_ways, values_relations, " \
                 " users_all, " \
-                "grids) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
+                "cells_nodes, cells_ways) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
 
         Osmium::Sqlite::Statement *statement_insert_into_tags = db->prepare("INSERT INTO tags (key, value, " \
                 "count_all, count_nodes, count_ways, count_relations) " \
                 "VALUES (?, ?, ?, ?, ?, ?);");
 
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
-        Osmium::Sqlite::Statement *statement_insert_into_key_combinations = db->prepare("INSERT INTO keypairs (key1, key2, " \
+        Osmium::Sqlite::Statement* statement_insert_into_key_combinations = db->prepare("INSERT INTO keypairs (key1, key2, " \
                 "count_all, count_nodes, count_ways, count_relations) " \
                 "VALUES (?, ?, ?, ?, ?, ?);");
 #endif // TAGSTATS_COUNT_KEY_COMBINATIONS
 
-        Osmium::Sqlite::Statement *statement_update_meta = db->prepare("UPDATE source SET data_until=?");
+        Osmium::Sqlite::Statement* statement_update_meta = db->prepare("UPDATE source SET data_until=?");
 
         db->begin_transaction();
 
-        struct tm *tm = gmtime(&max_timestamp);
+        struct tm* tm = gmtime(&max_timestamp);
         static char max_timestamp_str[20]; // thats enough space for the timestamp generated from the pattern in the next line
         strftime(max_timestamp_str, sizeof(max_timestamp_str), "%Y-%m-%d %H:%M:%S", tm);
         statement_update_meta->bind_text(max_timestamp_str)->execute();
@@ -413,7 +467,8 @@ public:
 #else
             ->bind_int64(0)
 #endif // TAGSTATS_COUNT_USERS
-            ->bind_int64(stat->node_distribution.get_cells()) // column: grids
+            ->bind_int64(stat->cells.nodes())      // column: cells_nodes
+            ->bind_int64(stat->cells.ways())       // column: cells_ways
             ->execute();
 
 #ifdef TAGSTATS_COUNT_KEY_COMBINATIONS
diff --git a/tagstats/test_tagstats.sh b/tagstats/test_tagstats.sh
new file mode 100755
index 0000000..99bd12b
--- /dev/null
+++ b/tagstats/test_tagstats.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+#
+#  test_tagstats.sh
+#
+
+set -e
+set -x
+
+DATABASE=taginfo-db.db
+OSMFILE=$1
+
+rm -f $DATABASE
+
+sqlite3 $DATABASE <../sources/init.sql
+sqlite3 $DATABASE <../sources/db/pre.sql
+
+ulimit -c 1000000000
+rm -f core
+
+#./tagstats --left=5.5 --bottom=47 --right=15 --top=55 --width=200 --height=320 $OSMFILE
+./tagstats $OSMFILE
+
diff --git a/web/lib/api/db.rb b/web/lib/api/db.rb
index a977965..aafb763 100644
--- a/web/lib/api/db.rb
+++ b/web/lib/api/db.rb
@@ -201,7 +201,7 @@ class Taginfo < Sinatra::Base
     }) do
         key = params[:key]
         content_type :png
-        @db.select('SELECT png FROM db.key_distributions').
+        @db.select('SELECT nodes FROM db.key_distributions').
             condition('key = ?', key).
             get_first_value()
     end