Skip to content

Commit

Permalink
Removed non-METATILE code as it no longer seems to work
Browse files Browse the repository at this point in the history
Remove unused reference to METATILEFALLBACK
Rewrote `meta tile`/`meta-tile` to `metatile`
  • Loading branch information
hummeltech committed Jan 30, 2024
1 parent 38fb906 commit 7884376
Show file tree
Hide file tree
Showing 14 changed files with 26 additions and 149 deletions.
2 changes: 1 addition & 1 deletion README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ Apache file serving mechanisms to provide:
5) Tile expiry. It estimates when the tile is next likely to be rendered and adds the appropriate HTTP cache expiry headers. This is a configurable heuristic.

To avoid problems with directories becoming too large and to avoid
too many tiny files. They store the rendered tiles in "meta tiles" in a
too many tiny files. They store the rendered tiles in "metatiles" in a
special hashed directory structure. These combine 8x8 actual tiles into a
single metatile file. This is a more efficient use of disk space and inodes.

Expand Down
15 changes: 1 addition & 14 deletions includes/render_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -102,28 +102,15 @@
// default for number of rendering threads
#define NUM_THREADS (4)

// Use this to enable meta-tiles which will render NxN tiles at once
// Metatiles will render NxN tiles at once
// Note: This should be a power of 2 (2, 4, 8, 16 ...)
#define METATILE (8)
//#undef METATILE

//Fallback to standard tiles if meta tile doesn't exist
//Legacy - not needed on new installs
//#undef METATILEFALLBACK

// Metatiles are much larger in size so we don't need big queues to handle large areas
#ifdef METATILE
#define QUEUE_MAX (64)
#define REQ_LIMIT (256)
#define DIRTY_LIMIT (8000)

#else
#define QUEUE_MAX (1024)
#define REQ_LIMIT (512)
#define DIRTY_LIMIT (10000)
#define HASHIDX_SIZE 22123
#endif

// Penalty for client making an invalid request (in seconds)
#define CLIENT_PENALTY (3)

Expand Down
7 changes: 2 additions & 5 deletions includes/store_file_utils.h
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,15 @@ extern "C" {
int mkdirp(const char *path);

/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* The two must both agree on the file layout for metatiling
* to work
*/
void xyz_to_path(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);

int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px, int *py, int *pz);

#ifdef METATILE
/* New meta-tile storage functions */
/* Returns the path to the meta-tile and the offset within the meta-tile */
/* Returns the path to the metatile and the offset within the metatile */
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
#endif

#ifdef __cplusplus
}
Expand Down
61 changes: 1 addition & 60 deletions src/gen_tile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,7 @@ using namespace mapnik;
#define RAD_TO_DEG (180/M_PI)
#endif

#ifdef METATILE
#define RENDER_SIZE (256 * (METATILE + 1))
#else
#define RENDER_SIZE (512)
#endif

extern int foreground;

Expand Down Expand Up @@ -245,7 +241,6 @@ static int check_xyz(int x, int y, int z, struct xmlmapconfig * map)
return !oob;
}

#ifdef METATILE
mapnik::box2d<double> tile2prjbounds(struct projectionconfig * prj, int x, int y, int z)
{

Expand Down Expand Up @@ -299,7 +294,7 @@ static enum protoCmd render(struct xmlmapconfig * map, int x, int y, int z, char
return cmdNotDone;
}

// Split the meta tile into an NxN grid of tiles
// Split the metatile into an NxN grid of tiles
unsigned int xx, yy;

for (yy = 0; yy < render_size_ty; yy++) {
Expand All @@ -312,52 +307,6 @@ static enum protoCmd render(struct xmlmapconfig * map, int x, int y, int z, char

return cmdDone; // OK
}
#else //METATILE
static enum protoCmd render(Map &m, const char *tile_dir, char *xmlname, projection &prj, int x, int y, int z, char* outputFormat)
{
char filename[PATH_MAX];
char tmp[PATH_MAX];
double p0x = x * 256.0;
double p0y = (y + 1) * 256.0;
double p1x = (x + 1) * 256.0;
double p1y = y * 256.0;

tiling.fromPixelToLL(p0x, p0y, z);
tiling.fromPixelToLL(p1x, p1y, z);

prj.forward(p0x, p0y);
prj.forward(p1x, p1y);

mapnik::box2d<double> bbox(p0x, p0y, p1x, p1y);
bbox.width(bbox.width() * 2);
bbox.height(bbox.height() * 2);
m.zoomToBox(bbox);

mapnik::image_32 buf(RENDER_SIZE, RENDER_SIZE);
mapnik::agg_renderer<mapnik::image_32> ren(m, buf);
ren.apply();

xyz_to_path(filename, sizeof(filename), tile_dir, xmlname, x, y, z);

if (mkdirp(filename)) {
return cmdNotDone;
}

snprintf(tmp, sizeof(tmp), "%s.tmp", filename);

mapnik::image_view<mapnik::image_data_32> vw(128, 128, 256, 256, buf.data());
g_logger(G_LOG_LEVEL_DEBUG, "Render %i %i %i %s", z, x, y, filename)
mapnik::save_to_file(vw, tmp, outputFormat);

if (rename(tmp, filename)) {
perror(tmp);
return cmdNotDone;
}

return cmdDone; // OK
}
#endif //METATILE


void render_init(const char *plugins_dir, const char* font_dir, int font_dir_recurse)
{
Expand Down Expand Up @@ -449,7 +398,6 @@ void *render_thread(void * arg)

if (item) {
struct protocol *req = &item->req;
#ifdef METATILE
// At very low zoom the whole world may be smaller than METATILE
unsigned int size = MIN(METATILE, 1 << req->z);

Expand Down Expand Up @@ -501,13 +449,6 @@ void *render_thread(void * arg)
request_exit();
}
}

#else //METATILE
ret = render(maps[i].map, maps[i].tile_dir, req->xmlname, maps[i].prj, req->x, req->y, req->z, maps[i].output_format);
#ifdef HTCP_EXPIRE_CACHE
cache_expire(maps[i].htcpsock, maps[i].host, maps[i].xmluri, req->x, req->y, req->z);
#endif
#endif //METATILE
} else {
g_logger(G_LOG_LEVEL_WARNING, "Received request for map layer %s is outside of acceptable bounds z(%i), x(%i), y(%i)",
req->xmlname, req->z, req->x, req->y);
Expand Down
2 changes: 1 addition & 1 deletion src/metatile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ const std::string metaTile::get(int x, int y)
return tile[x][y];
}

// Returns the offset within the meta-tile index table
// Returns the offset within the metatile index table
int metaTile::xyz_to_meta_offset(int x, int y, int z)
{
unsigned char mask = METATILE - 1;
Expand Down
23 changes: 6 additions & 17 deletions src/render_expired.c
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ const char * tile_dir_default = RENDERD_TILE_DIR;

// macros handling our tile marking arrays (these are essentially bit arrays
// that have one bit for each tile on the repsective zoom level; since we only
// need them for meta tile levels, even if someone were to render level 20,
// need them for metatile levels, even if someone were to render level 20,
// we'd still only use 4^17 bits = 2 GB RAM (plus a little for the lower zoom
// levels) - this saves us the hassle of working with a tree structure.

Expand All @@ -52,16 +52,6 @@ const char * tile_dir_default = RENDERD_TILE_DIR;
#define SET_TILE_REQUESTED(z,x,y) \
tile_requested[z][((x)*twopow[z]+(y))/(8*sizeof(int))] |= (0x01 << (((x)*twopow[z]+(y))%(8*sizeof(int))));


#ifndef METATILE
#warning("render_expired not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_expired not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

// tile marking arrays
unsigned int **tile_requested;

Expand Down Expand Up @@ -112,7 +102,7 @@ int main(int argc, char **argv)
char name[PATH_MAX];

// excess_zoomlevels is how many zoom levels at the large end
// we can ignore because their tiles will share one meta tile.
// we can ignore because their tiles will share one metatile.
// with the default METATILE==8 this is 3.
int excess_zoomlevels = 0;
int mt = METATILE;
Expand Down Expand Up @@ -399,7 +389,7 @@ int main(int argc, char **argv)
{
gettimeofday(&end, NULL);
printf("\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand Down Expand Up @@ -443,16 +433,15 @@ int main(int argc, char **argv)

gettimeofday(&end, NULL);
printf("\nTotal for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
printf("Total tiles in input: %d\n", num_read);
printf("Total tiles expanded from input: %d\n", num_all);
printf("Total meta tiles deleted: %d\n", num_unlink);
printf("Total meta tiles touched: %d\n", num_touch);
printf("Total metatiles deleted: %d\n", num_unlink);
printf("Total metatiles touched: %d\n", num_touch);
printf("Total tiles ignored (not on disk): %d\n", num_ignore);

return 0;
}
#endif
14 changes: 2 additions & 12 deletions src/render_list.c
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,6 @@

const char * tile_dir_default = RENDERD_TILE_DIR;

#ifndef METATILE
#warning("render_list not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_list not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

static int minZoom = 0;
static int maxZoom = MAX_ZOOM;
static int verbose = 0;
Expand Down Expand Up @@ -364,7 +355,7 @@ int main(int argc, char **argv)
if (!(num_render % 10)) {
gettimeofday(&end, NULL);
printf("\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, (num_render) * METATILE * METATILE);
Expand Down Expand Up @@ -398,7 +389,7 @@ int main(int argc, char **argv)
printf("\n*****************************************************\n");
printf("*****************************************************\n");
printf("Total for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand All @@ -408,4 +399,3 @@ int main(int argc, char **argv)

return 0;
}
#endif
12 changes: 1 addition & 11 deletions src/render_old.c
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,6 @@

const char * tile_dir_default = RENDERD_TILE_DIR;

#ifndef METATILE
#warning("render_old not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "render_old not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

#define INILINE_MAX 256
static int minZoom = 0;
static int maxZoom = MAX_ZOOM;
Expand Down Expand Up @@ -400,7 +391,7 @@ int main(int argc, char **argv)

gettimeofday(&end, NULL);
printf("\nTotal for all tiles rendered\n");
printf("Meta tiles rendered: ");
printf("Metatiles rendered: ");
display_rate(start, end, num_render);
printf("Total tiles rendered: ");
display_rate(start, end, num_render * METATILE * METATILE);
Expand All @@ -409,4 +400,3 @@ int main(int argc, char **argv)

return 0;
}
#endif
10 changes: 0 additions & 10 deletions src/render_speedtest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,6 @@
#define DEG_TO_RAD (M_PI/180)
#define RAD_TO_DEG (180/M_PI)

#ifndef METATILE
#warning("Speed test not implemented for non-metatile mode. Feel free to submit fix")
int main(int argc, char **argv)
{
fprintf(stderr, "Speed test not implemented for non-metatile mode. Feel free to submit fix!\n");
return -1;
}
#else

#if 1
static double boundx0 = -0.5;
static double boundy0 = 51.25;
Expand Down Expand Up @@ -295,4 +286,3 @@ int main(int argc, char **argv)

return ret;
}
#endif
5 changes: 0 additions & 5 deletions src/renderd.c
Original file line number Diff line number Diff line change
Expand Up @@ -168,17 +168,12 @@ enum protoCmd rx_request(struct protocol *req, int fd)
item->duplicates = NULL;
item->fd = (req->cmd == cmdDirty) ? FD_INVALID : fd;

#ifdef METATILE
/* Round down request co-ordinates to the neareast N (should be a power of 2)
* Note: request path is no longer consistent but this will be recalculated
* when the metatile is being rendered.
*/
item->mx = item->req.x & ~(METATILE - 1);
item->my = item->req.y & ~(METATILE - 1);
#else
item->mx = item->req.x;
item->my = item->req.y;
#endif

return request_queue_add_request(render_request_queue, item);
}
Expand Down
4 changes: 2 additions & 2 deletions src/store_file.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
* along with this program; If not, see http://www.gnu.org/licenses/.
*/

/* Meta-tile optimised file storage
/* Metatile optimised file storage
*
* Instead of storing each individual tile as a file,
* bundle the 8x8 meta tile into a special meta-file.
* bundle the 8x8 metatile into a special meta-file.
* This reduces the Inode usage and more efficient
* utilisation of disk space.
*/
Expand Down
12 changes: 5 additions & 7 deletions src/store_file_utils.c
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ int mkdirp(const char *path)


/* File path hashing. Used by both mod_tile and render daemon
* The two must both agree on the file layout for meta-tiling
* The two must both agree on the file layout for metatiling
* to work
*/

Expand Down Expand Up @@ -191,20 +191,19 @@ int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px
#endif
}

#ifdef METATILE
// Returns the path to the meta-tile and the offset within the meta-tile
// Returns the path to the metatile and the offset within the metatile
int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z)
{
return xyzo_to_meta(path, len, tile_dir, xmlconfig, "", x, y, z);
}

// Returns the path to the meta-tile and the offset within the meta-tile
// Returns the path to the metatile and the offset within the metatile
int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z)
{
unsigned char i, hash[5], offset, mask;

// Each meta tile winds up in its own file, with several in each leaf directory
// the .meta tile name is beasd on the sub-tile at (0,0)
// Each metatile winds up in its own file, with several in each leaf directory
// the .meta file name is based on the sub-tile at (0,0)
mask = METATILE - 1;
offset = (x & mask) * METATILE + (y & mask);
x &= ~mask;
Expand Down Expand Up @@ -235,4 +234,3 @@ int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlco
#endif
return offset;
}
#endif
Loading

0 comments on commit 7884376

Please sign in to comment.