diff --git a/CHANGELOG.md b/CHANGELOG.md index 899e5ce4d..bf004ccdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.11.7 + +* Keep metadata together with geometry for features that don't span many tiles, + to avoid extra memory load from indexing into a separate metadata file + ## 1.11.6 * Reduce the size of critical data structures to reduce dynamic memory use diff --git a/geojson.cpp b/geojson.cpp index ce1ea0868..4d1866a91 100644 --- a/geojson.cpp +++ b/geojson.cpp @@ -36,6 +36,7 @@ extern "C" { #include "mbtiles.hpp" #include "geojson.hpp" #include "geometry.hpp" +#include "options.hpp" #define GEOM_POINT 0 /* array of positions */ #define GEOM_MULTIPOINT 1 /* array of arrays of positions */ @@ -62,10 +63,12 @@ static int mb_geometry[GEOM_TYPES] = { VT_POINT, VT_POINT, VT_LINE, VT_LINE, VT_POLYGON, VT_POLYGON, }; -void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { +long long parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FILE *out, int op, const char *fname, int line, long long *wx, long long *wy, int *initialized, unsigned *initial_x, unsigned *initial_y) { + long long g = 0; + if (j == NULL || j->type != JSON_ARRAY) { fprintf(stderr, "%s:%d: expected array for type %d\n", fname, line, t); - return; + return g; } int within = geometry_within[t]; @@ -80,7 +83,7 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL } } - parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y); + g += parse_geometry(within, j->array[i], bbox, fpos, out, op, fname, line, wx, wy, initialized, initial_x, initial_y); } } else { if (j->length >= 2 && j->array[0]->type == JSON_NUMBER && j->array[1]->type == JSON_NUMBER) { @@ -98,19 +101,17 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL } } - if (bbox != NULL) { - if (x < bbox[0]) { - bbox[0] = x; - } - if (y < bbox[1]) { - bbox[1] = y; - } - if (x > bbox[2]) { - bbox[2] = x; - } - if (y > bbox[3]) { - bbox[3] = y; - } + if (x < bbox[0]) { + bbox[0] = x; + } + if (y < bbox[1]) { + bbox[1] = y; + } + if (x > bbox[2]) { + bbox[2] = x; + } + if (y > bbox[3]) { + bbox[3] = y; } if (!*initialized) { @@ -134,6 +135,7 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL serialize_long_long(out, (y >> geometry_scale) - (*wy >> geometry_scale), fpos, fname); *wx = x; *wy = y; + g++; } else { fprintf(stderr, "%s:%d: malformed point\n", fname, line); } @@ -151,9 +153,11 @@ void parse_geometry(int t, json_object *j, long long *bbox, long long *fpos, FIL serialize_byte(out, VT_CLOSEPATH, fpos, fname); } + + return g; } -int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys) { +int serialize_geometry(json_object *geometry, json_object *properties, const char *reading, int line, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, const char *fname, int basezoom, int layer, double droprate, long long *file_bbox, json_object *tippecanoe, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom) { json_object *geometry_type = json_hash_get(geometry, "type"); if (geometry_type == NULL) { static int warned = 0; @@ -208,7 +212,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha } } - long long bbox[] = {UINT_MAX, UINT_MAX, 0, 0}; + long long bbox[] = {LLONG_MAX, LLONG_MAX, LLONG_MIN, LLONG_MIN}; int nprop = 0; if (properties != NULL && properties->type == JSON_HASH) { @@ -269,15 +273,6 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha } } - for (i = 0; i < m; i++) { - serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); - serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname); - - if (mustfree[i]) { - free((void *) metaval[i]); - } - } - long long geomstart = *geompos; serialize_byte(geomfile, mb_geometry[t], geompos, fname); @@ -292,12 +287,56 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha } serialize_int(geomfile, segment, geompos, fname); - serialize_long_long(geomfile, metastart, geompos, fname); - serialize_int(geomfile, m, geompos, fname); long long wx = *initial_x, wy = *initial_y; - parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y); + long long g = parse_geometry(t, coordinates, bbox, geompos, geomfile, VT_MOVETO, fname, line, &wx, &wy, initialized, initial_x, initial_y); serialize_byte(geomfile, VT_END, geompos, fname); + bool inline_meta = true; + // Don't inline metadata for features that will span several tiles at maxzoom + if (g > 0 && (bbox[2] < bbox[0] || bbox[3] < bbox[1])) { + fprintf(stderr, "Internal error: impossible feature bounding box %llx,%llx,%llx,%llx\n", bbox[0], bbox[1], bbox[2], bbox[3]); + } + if (bbox[2] - bbox[0] > (2LL << (32 - maxzoom)) || bbox[3] - bbox[1] > (2LL << (32 - maxzoom))) { + inline_meta = false; + + if (prevent[P_CLIPPING]) { + static volatile long long warned = 0; + long long extent = ((bbox[2] - bbox[0]) / ((1LL << (32 - maxzoom)) + 1)) * ((bbox[3] - bbox[1]) / ((1LL << (32 - maxzoom)) + 1)); + if (extent > warned) { + fprintf(stderr, "Warning: %s:%d: Large unclipped (-pc) feature may be duplicated across %lld tiles\n", fname, line, extent); + warned = extent; + + if (extent > 10000) { + fprintf(stderr, "Exiting because this can't be right.\n"); + exit(EXIT_FAILURE); + } + } + } + } + + serialize_int(geomfile, m, geompos, fname); + if (inline_meta) { + serialize_long_long(geomfile, -1, geompos, fname); + + for (i = 0; i < m; i++) { + serialize_long_long(geomfile, addpool(poolfile, treefile, metakey[i], VT_STRING), geompos, fname); + serialize_long_long(geomfile, addpool(poolfile, treefile, metaval[i], metatype[i]), geompos, fname); + } + } else { + serialize_long_long(geomfile, metastart, geompos, fname); + + for (i = 0; i < m; i++) { + serialize_long_long(metafile, addpool(poolfile, treefile, metakey[i], VT_STRING), metapos, fname); + serialize_long_long(metafile, addpool(poolfile, treefile, metaval[i], metatype[i]), metapos, fname); + } + } + + for (i = 0; i < m; i++) { + if (mustfree[i]) { + free((void *) metaval[i]); + } + } + /* * Note that feature_minzoom for lines is the dimension * of the geometry in world coordinates, but @@ -366,7 +405,7 @@ int serialize_geometry(json_object *geometry, json_object *properties, const cha return 1; } -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys) { +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom) { long long found_hashes = 0; long long found_features = 0; long long found_geometries = 0; @@ -431,7 +470,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se } found_geometries++; - serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys); + serialize_geometry(j, NULL, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, NULL, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); json_free(j); continue; } @@ -466,10 +505,10 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se if (geometries != NULL) { size_t g; for (g = 0; g < geometries->length; g++) { - serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys); + serialize_geometry(geometries->array[g], properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); } } else { - serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys); + serialize_geometry(geometry, properties, reading, jp->line, layer_seq, progress_seq, metapos, geompos, indexpos, exclude, include, exclude_all, metafile, geomfile, indexfile, poolfile, treefile, fname, basezoom, layer, droprate, file_bbox, tippecanoe, segment, initialized, initial_x, initial_y, readers, file_keys, maxzoom); } json_free(j); @@ -481,7 +520,7 @@ void parse_json(json_pull *jp, const char *reading, volatile long long *layer_se void *run_parse_json(void *v) { struct parse_json_args *pja = (struct parse_json_args *) v; - parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->file_keys); + parse_json(pja->jp, pja->reading, pja->layer_seq, pja->progress_seq, pja->metapos, pja->geompos, pja->indexpos, pja->exclude, pja->include, pja->exclude_all, pja->metafile, pja->geomfile, pja->indexfile, pja->poolfile, pja->treefile, pja->fname, pja->basezoom, pja->layer, pja->droprate, pja->file_bbox, pja->segment, pja->initialized, pja->initial_x, pja->initial_y, pja->readers, pja->file_keys, pja->maxzoom); return NULL; } diff --git a/geojson.hpp b/geojson.hpp index cec9820d5..5c48c2374 100644 --- a/geojson.hpp +++ b/geojson.hpp @@ -25,8 +25,9 @@ struct parse_json_args { unsigned *initial_y; struct reader *readers; std::set *file_keys; + int maxzoom; }; struct json_pull *json_begin_map(char *map, long long len); -void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys); +void parse_json(json_pull *jp, const char *reading, volatile long long *layer_seq, volatile long long *progress_seq, long long *metapos, long long *geompos, long long *indexpos, std::set *exclude, std::set *include, int exclude_all, FILE *metafile, FILE *geomfile, FILE *indexfile, struct memfile *poolfile, struct memfile *treefile, char *fname, int basezoom, int layer, double droprate, long long *file_bbox, int segment, int *initialized, unsigned *initial_x, unsigned *initial_y, struct reader *readers, std::set *file_keys, int maxzoom); void *run_parse_json(void *v); diff --git a/main.cpp b/main.cpp index d5d5fde59..bcb7f4914 100644 --- a/main.cpp +++ b/main.cpp @@ -56,8 +56,8 @@ static int min_detail = 7; int quiet = 0; int geometry_scale = 0; -static int prevent[256]; -static int additional[256]; +int prevent[256]; +int additional[256]; struct source { std::string layer; @@ -298,7 +298,7 @@ void *run_sort(void *v) { return NULL; } -void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys) { +void do_read_parallel(char *map, long long len, long long initial_offset, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { long long segs[CPUS + 1]; segs[0] = 0; segs[CPUS] = len; @@ -354,6 +354,7 @@ void do_read_parallel(char *map, long long len, long long initial_offset, const pja[i].initial_y = &initial_y[i]; pja[i].readers = reader; pja[i].file_keys = &file_subkeys[i]; + pja[i].maxzoom = maxzoom; if (pthread_create(&pthreads[i], NULL, run_parse_json, &pja[i]) != 0) { perror("pthread_create"); @@ -422,7 +423,7 @@ void *run_read_parallel(void *v) { } madvise(map, a->len, MADV_RANDOM); // sequential, but from several pointers at once - do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys); + do_read_parallel(map, a->len, a->offset, a->reading, a->reader, a->progress_seq, a->exclude, a->include, a->exclude_all, a->fname, a->basezoom, a->source, a->nlayers, a->droprate, a->initialized, a->initial_x, a->initial_y, a->file_keys, a->maxzoom); madvise(map, a->len, MADV_DONTNEED); if (munmap(map, a->len) != 0) { @@ -439,7 +440,7 @@ void *run_read_parallel(void *v) { return NULL; } -void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys) { +void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile int *is_parsing, pthread_t *parallel_parser, const char *reading, struct reader *reader, volatile long long *progress_seq, std::set *exclude, std::set *include, int exclude_all, char *fname, int basezoom, int source, int nlayers, double droprate, int *initialized, unsigned *initial_x, unsigned *initial_y, std::set *file_keys, int maxzoom) { // This has to kick off an intermediate thread to start the parser threads, // so the main thread can get back to reading the next input stage while // the intermediate thread waits for the completion of the parser threads. @@ -473,6 +474,7 @@ void start_parsing(int fd, FILE *fp, long long offset, long long len, volatile i rpa->initial_x = initial_x; rpa->initial_y = initial_y; rpa->file_keys = file_keys; + rpa->maxzoom = maxzoom; if (pthread_create(parallel_parser, NULL, run_read_parallel, rpa) != 0) { perror("pthread_create"); @@ -1015,7 +1017,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } if (map != NULL && map != MAP_FAILED) { - do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]); + do_read_parallel(map, st.st_size - off, overall_offset, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom); overall_offset += st.st_size - off; checkdisk(reader, CPUS); @@ -1081,7 +1083,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } fflush(readfp); - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom); initial_offset += ahead; overall_offset += ahead; @@ -1117,7 +1119,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, fflush(readfp); if (ahead > 0) { - start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0]); + start_parsing(readfd, readfp, initial_offset, ahead, &is_parsing, ¶llel_parser, reading.c_str(), reader, &progress_seq, exclude, include, exclude_all, fname, basezoom, source, nlayers, droprate, initialized, initial_x, initial_y, &file_keys[source < nlayers ? source : 0], maxzoom); if (pthread_join(parallel_parser, NULL) != 0) { perror("pthread_join"); @@ -1131,7 +1133,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, long long layer_seq = overall_offset; json_pull *jp = json_begin_file(fp); - parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0]); + parse_json(jp, reading.c_str(), &layer_seq, &progress_seq, &reader[0].metapos, &reader[0].geompos, &reader[0].indexpos, exclude, include, exclude_all, reader[0].metafile, reader[0].geomfile, reader[0].indexfile, reader[0].poolfile, reader[0].treefile, fname, basezoom, source < nlayers ? source : 0, droprate, reader[0].file_bbox, 0, &initialized[0], &initial_x[0], &initial_y[0], reader, &file_keys[source < nlayers ? source : 0], maxzoom); json_end(jp); overall_offset = layer_seq; checkdisk(reader, CPUS); @@ -1391,7 +1393,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, progress_seq = indexpos / sizeof(struct index); if (!quiet) { - fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); + fprintf(stderr, "%lld features, %lld bytes of geometry, %lld bytes of separate metadata, %lld bytes of string pool\n", progress_seq, geompos, metapos, poolpos); } if (indexpos == 0) { @@ -1597,7 +1599,7 @@ int read_input(std::vector &sources, char *fname, const char *layername, } unsigned midx = 0, midy = 0; - int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, prevent, additional, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y); + int written = traverse_zooms(fd, size, meta, stringpool, &midx, &midy, layernames, maxzoom, minzoom, basezoom, outdb, droprate, buffer, fname, tmpdir, gamma, nlayers, full_detail, low_detail, min_detail, meta_off, pool_off, initial_x, initial_y); if (maxzoom != written) { fprintf(stderr, "\n\n\n*** NOTE TILES ONLY COMPLETE THROUGH ZOOM %d ***\n\n\n", written); diff --git a/options.hpp b/options.hpp index 3928af50d..d8c0a54ba 100644 --- a/options.hpp +++ b/options.hpp @@ -15,3 +15,6 @@ #define P_POLYGON_SPLIT ((int) 'p') #define P_CLIPPING ((int) 'c') #define P_DUPLICATION ((int) 'D') + +extern int prevent[256]; +extern int additional[256]; diff --git a/tile.cpp b/tile.cpp index b54a4e62d..4909d1b85 100644 --- a/tile.cpp +++ b/tile.cpp @@ -57,13 +57,15 @@ bool draws_something(drawvec &geom) { return false; } -int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2); +int metacmp(int m1, const std::vector &keys1, const std::vector &values1, char *stringpool1, int m2, const std::vector &keys2, const std::vector &values2, char *stringpool2); int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2); static int is_integer(const char *s, long long *v); struct coalesce { char *meta; char *stringpool; + std::vector keys; + std::vector values; drawvec geom; unsigned long long index; unsigned long long index2; @@ -97,10 +99,7 @@ int coalcmp(const void *v1, const void *v2) { return cmp; } - char *m1 = c1->meta; - char *m2 = c2->meta; - - return metacmp(c1->m, &m1, c1->stringpool, c2->m, &m2, c2->stringpool); + return metacmp(c1->m, c1->keys, c1->values, c1->stringpool, c2->m, c2->keys, c2->values, c2->stringpool); } int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) { @@ -123,10 +122,7 @@ int coalindexcmp(const struct coalesce *c1, const struct coalesce *c2) { return cmp; } -mvt_value retrieve_string(char **f, char *stringpool, int *otype) { - long long off; - deserialize_long_long(f, &off); - +mvt_value retrieve_string(long long off, char *stringpool, int *otype) { int type = stringpool[off]; char *s = stringpool + off + 1; @@ -160,18 +156,18 @@ mvt_value retrieve_string(char **f, char *stringpool, int *otype) { return tv; } -void decode_meta(int m, char **meta, char *stringpool, mvt_layer &layer, mvt_feature &feature) { +void decode_meta(int m, std::vector &metakeys, std::vector &metavals, char *stringpool, mvt_layer &layer, mvt_feature &feature) { int i; for (i = 0; i < m; i++) { int otype; - mvt_value key = retrieve_string(meta, stringpool, NULL); - mvt_value value = retrieve_string(meta, stringpool, &otype); + mvt_value key = retrieve_string(metakeys[i], stringpool, NULL); + mvt_value value = retrieve_string(metavals[i], stringpool, &otype); layer.tag(feature, key.string_value, value); } } -int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char *stringpool2) { +int metacmp(int m1, const std::vector &keys1, const std::vector &values1, char *stringpool1, int m2, const std::vector &keys2, const std::vector &values2, char *stringpool2) { // XXX // Ideally this would make identical features compare the same lexically // even if their attributes were declared in different orders in different instances. @@ -179,8 +175,8 @@ int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char int i; for (i = 0; i < m1 && i < m2; i++) { - mvt_value key1 = retrieve_string(meta1, stringpool1, NULL); - mvt_value key2 = retrieve_string(meta2, stringpool2, NULL); + mvt_value key1 = retrieve_string(keys1[i], stringpool1, NULL); + mvt_value key2 = retrieve_string(keys2[i], stringpool2, NULL); if (key1.string_value < key2.string_value) { return -1; @@ -188,13 +184,11 @@ int metacmp(int m1, char **meta1, char *stringpool1, int m2, char **meta2, char return 1; } - long long off1; - deserialize_long_long(meta1, &off1); + long long off1 = values1[i]; int type1 = stringpool1[off1]; char *s1 = stringpool1 + off1 + 1; - long long off2; - deserialize_long_long(meta2, &off2); + long long off2 = values2[i]; int type2 = stringpool2[off2]; char *s2 = stringpool2 + off2 + 1; @@ -265,7 +259,7 @@ struct sll { } }; -void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m) { +void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, unsigned tx, unsigned ty, int buffer, int line_detail, int *within, long long *geompos, FILE **geomfile, const char *fname, signed char t, int layer, long long metastart, signed char feature_minzoom, int child_shards, int max_zoom_increment, long long seq, int tippecanoe_minzoom, int tippecanoe_maxzoom, int segment, unsigned *initial_x, unsigned *initial_y, int m, std::vector &metakeys, std::vector &metavals) { if (geom.size() > 0 && nextzoom <= maxzoom) { int xo, yo; int span = 1 << (nextzoom - z); @@ -347,8 +341,6 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u serialize_int(geomfile[j], tippecanoe_maxzoom, geompos, fname); } serialize_int(geomfile[j], segment, &geompos[j], fname); - serialize_long_long(geomfile[j], metastart, &geompos[j], fname); - serialize_int(geomfile[j], m, &geompos[j], fname); long long wx = initial_x[segment], wy = initial_y[segment]; for (size_t u = 0; u < geom.size(); u++) { @@ -363,6 +355,16 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u } serialize_byte(geomfile[j], VT_END, &geompos[j], fname); + + serialize_int(geomfile[j], m, &geompos[j], fname); + serialize_long_long(geomfile[j], metastart, &geompos[j], fname); + if (metastart < 0) { + for (int i = 0; i < m; i++) { + serialize_long_long(geomfile[j], metakeys[i], &geompos[j], fname); + serialize_long_long(geomfile[j], metavals[i], &geompos[j], fname); + } + } + serialize_byte(geomfile[j], feature_minzoom, &geompos[j], fname); } } @@ -372,9 +374,9 @@ void rewrite(drawvec &geom, int z, int nextzoom, int maxzoom, long long *bbox, u struct partial { std::vector geoms; + std::vector keys; + std::vector values; char *meta; - int *prevent; - int *additional; long long layer; long long original_seq; unsigned long long index; @@ -445,8 +447,6 @@ void *partial_feature_worker(void *v) { signed char t = (*partials)[i].t; int z = (*partials)[i].z; int line_detail = (*partials)[i].line_detail; - int *prevent = (*partials)[i].prevent; - int *additional = (*partials)[i].additional; int maxzoom = (*partials)[i].maxzoom; double area = 0; @@ -557,7 +557,7 @@ int manage_gap(unsigned long long index, unsigned long long *previndex, double s return 0; } -long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, std::vector *layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int *prevent, int *additional, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { +long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *stringpool, int z, unsigned tx, unsigned ty, int detail, int min_detail, int basezoom, std::vector *layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int minzoom, int maxzoom, double todo, volatile long long *along, double gamma, int nlayers, int child_shards, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y, volatile int *running) { int line_detail; double fraction = 1; @@ -649,15 +649,37 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s int segment; deserialize_int_io(geoms, &segment, geompos_in); - long long metastart; - int m; - deserialize_long_long_io(geoms, &metastart, geompos_in); - deserialize_int_io(geoms, &m, geompos_in); - char *meta = metabase + metastart + meta_off[segment]; long long bbox[4]; drawvec geom = decode_geometry(geoms, geompos_in, z, tx, ty, line_detail, bbox, initial_x[segment], initial_y[segment]); + long long metastart; + int m; + deserialize_int_io(geoms, &m, geompos_in); + deserialize_long_long_io(geoms, &metastart, geompos_in); + char *meta = NULL; + std::vector metakeys, metavals; + + if (metastart >= 0) { + meta = metabase + metastart + meta_off[segment]; + + for (int i = 0; i < m; i++) { + long long k, v; + deserialize_long_long(&meta, &k); + deserialize_long_long(&meta, &v); + metakeys.push_back(k); + metavals.push_back(v); + } + } else { + for (int i = 0; i < m; i++) { + long long k, v; + deserialize_long_long_io(geoms, &k, geompos_in); + deserialize_long_long_io(geoms, &v, geompos_in); + metakeys.push_back(k); + metavals.push_back(v); + } + } + signed char feature_minzoom; deserialize_byte_io(geoms, &feature_minzoom, geompos_in); @@ -752,7 +774,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s } if (line_detail == detail && fraction == 1) { /* only write out the next zoom once, even if we retry */ - rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom, segment, initial_x, initial_y, m); + rewrite(geom, z, nextzoom, maxzoom, bbox, tx, ty, buffer, line_detail, within, geompos, geomfile, fname, t, layer, metastart, feature_minzoom, child_shards, max_zoom_increment, original_seq, tippecanoe_minzoom, tippecanoe_maxzoom, segment, initial_x, initial_y, m, metakeys, metavals); } if (z < minzoom) { @@ -815,9 +837,9 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s p.reduced = reduced; p.z = z; p.line_detail = line_detail; - p.prevent = prevent; - p.additional = additional; p.maxzoom = maxzoom; + p.keys = metakeys; + p.values = metavals; partials.push_back(p); } } @@ -876,6 +898,8 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s c.m = partials[i].m; c.meta = partials[i].meta; c.stringpool = stringpool + pool_off[partials[i].segment]; + c.keys = partials[i].keys; + c.values = partials[i].values; features[layer].push_back(c); } @@ -970,7 +994,7 @@ long long write_tile(FILE *geoms, long long *geompos_in, char *metabase, char *s count += features[k][x].geom.size(); features[k][x].geom.clear(); - decode_meta(features[k][x].m, &features[k][x].meta, features[k][x].stringpool, layer, feature); + decode_meta(features[k][x].m, features[k][x].keys, features[k][x].values, features[k][x].stringpool, layer, feature); layer.features.push_back(feature); } @@ -1058,8 +1082,6 @@ struct write_tile_args { volatile long long *along; double gamma; int nlayers; - int *prevent; - int *additional; int child_shards; int *geomfd; off_t *geom_size; @@ -1116,7 +1138,7 @@ void *run_thread(void *vargs) { // fprintf(stderr, "%d/%u/%u\n", z, x, y); - long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->prevent, arg->additional, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running); + long long len = write_tile(geom, &geompos, arg->metabase, arg->stringpool, z, x, y, z == arg->maxzoom ? arg->full_detail : arg->low_detail, arg->min_detail, arg->basezoom, arg->layernames, arg->outdb, arg->droprate, arg->buffer, arg->fname, arg->geomfile, arg->minzoom, arg->maxzoom, arg->todo, arg->along, arg->gamma, arg->nlayers, arg->child_shards, arg->meta_off, arg->pool_off, arg->initial_x, arg->initial_y, arg->running); if (len < 0) { int *err = &arg->err; @@ -1167,7 +1189,7 @@ void *run_thread(void *vargs) { return NULL; } -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) { +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y) { int i; for (i = 0; i <= maxzoom; i++) { long long most = 0; @@ -1287,8 +1309,6 @@ int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpo args[thread].along = &along; // locked with var_lock args[thread].gamma = gamma; args[thread].nlayers = nlayers; - args[thread].prevent = prevent; - args[thread].additional = additional; args[thread].child_shards = TEMP_FILES / threads; args[thread].geomfd = geomfd; diff --git a/tile.hpp b/tile.hpp index 144f0a615..e19b837a5 100644 --- a/tile.hpp +++ b/tile.hpp @@ -1,5 +1,5 @@ -long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers, int *prevent, int *additional); +long long write_tile(char **geom, char *metabase, char *stringpool, unsigned *file_bbox, int z, unsigned x, unsigned y, int detail, int min_detail, int basezoom, char **layernames, sqlite3 *outdb, double droprate, int buffer, const char *fname, FILE **geomfile, int file_minzoom, int file_maxzoom, double todo, char *geomstart, long long along, double gamma, int nlayers); -int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int *prevent, int *additional, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); +int traverse_zooms(int *geomfd, off_t *geom_size, char *metabase, char *stringpool, unsigned *midx, unsigned *midy, std::vector &layernames, int maxzoom, int minzoom, int basezoom, sqlite3 *outdb, double droprate, int buffer, const char *fname, const char *tmpdir, double gamma, int nlayers, int full_detail, int low_detail, int min_detail, long long *meta_off, long long *pool_off, unsigned *initial_x, unsigned *initial_y); int manage_gap(unsigned long long index, unsigned long long *previndex, double scale, double gamma, double *gap); diff --git a/version.hpp b/version.hpp index 8a29be801..bc26cbfa6 100644 --- a/version.hpp +++ b/version.hpp @@ -1 +1 @@ -#define VERSION "tippecanoe v1.11.6\n" +#define VERSION "tippecanoe v1.11.7\n"