Addition of Amazon S3 tile store module

Makefile.am

@@ -6,10 +6,11 @@ endif
 ACLOCAL_AMFLAGS = -I m4
 
 AM_CPPFLAGS = $(PTHREAD_CFLAGS) -DSYSTEM_LIBINIPARSER=@SYSTEM_LIBINIPARSER@
+AM_CFLAGS = $(STORE_CFLAGS)
 
-STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c
-STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL)
-STORE_CPPFLAGS =
+STORE_SOURCES = src/store.c src/store_file.c src/store_file_utils.c src/store_memcached.c src/store_rados.c src/store_ro_http_proxy.c src/store_ro_composite.c src/store_null.c src/store_s3.c
+STORE_LDFLAGS = $(LIBMEMCACHED_LDFLAGS) $(LIBRADOS_LDFLAGS) $(LIBCURL) $(LIBS3_LDFLAGS)
+STORE_CFLAGS = $(LIBS3_CFLAGS)
 
 bin_PROGRAMS = renderd render_expired render_list render_speedtest render_old
 noinst_PROGRAMS = gen_tile_test
@@ -34,8 +35,8 @@ render_old_SOURCES = src/store_file_utils.c src/render_old.c src/sys_utils.c src
 render_old_LDADD = $(PTHREAD_CFLAGS)
 #convert_meta_SOURCES = src/dir_utils.c src/store.c src/convert_meta.c
 gen_tile_test_SOURCES = src/gen_tile_test.cpp src/metatile.cpp src/request_queue.c src/protocol_helper.c src/daemon.c src/daemon_compat.c src/gen_tile.cpp src/sys_utils.c src/cache_expire.c src/parameterize_style.cpp $(STORE_SOURCES)
-gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS)
-gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS)
+gen_tile_test_CFLAGS = -DMAIN_ALREADY_DEFINED $(PTHREAD_CFLAGS) $(STORE_CFLAGS)
+gen_tile_test_CXXFLAGS = $(MAPNIK_CFLAGS) $(STORE_CFLAGS)
 gen_tile_test_LDADD = $(PTHREAD_CFLAGS) $(MAPNIK_LDFLAGS) $(STORE_LDFLAGS) -liniparser
 if !SYSTEM_LIBINIPARSER
     gen_tile_test_SOURCES += iniparser3.0b/libiniparser.la
@@ -48,10 +49,10 @@ test: gen_tile_test
 	./gen_tile_test
 
 all-local:
-	$(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c  @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c
+	$(APXS) -c $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c  @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c
 
 install-mod_tile: 
 	mkdir -p $(DESTDIR)`$(APXS) -q LIBEXECDIR`
-	$(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_null.c
+	$(APXS) -S LIBEXECDIR=$(DESTDIR)`$(APXS) -q LIBEXECDIR` -c -i $(DEF_LDLIBS) $(AM_CFLAGS) -I@srcdir@/includes $(AM_LDFLAGS) $(STORE_LDFLAGS) @srcdir@/src/mod_tile.c @srcdir@/src/sys_utils.c @srcdir@/src/store.c @srcdir@/src/store_file.c @srcdir@/src/store_file_utils.c @srcdir@/src/store_memcached.c @srcdir@/src/store_rados.c @srcdir@/src/store_ro_http_proxy.c @srcdir@/src/store_ro_composite.c @srcdir@/src/store_s3.c @srcdir@/src/store_null.c
 
 

configure.ac

@@ -54,6 +54,46 @@ AC_CHECK_LIB(rados, rados_version, [
     AC_SUBST(LIBRADOS_LDFLAGS)
 ][])
 
+AC_ARG_WITH([libs3],
+    [AS_HELP_STRING([--with-libs3[=DIR]],[path to libs3])],
+    [
+    libs3_dir="$withval"
+    if test "$libs3_dir" != "no"; then
+      if test "$libs3_dir" != "yes"; then
+        AC_MSG_CHECKING([for libs3])
+        if test -f "$libs3_dir/include/libs3.h"; then
+          AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3])
+          LIBS3_LDFLAGS="-L$libs3_dir/lib -ls3"
+          AC_SUBST(LIBS3_LDFLAGS)
+          LIBS3_CFLAGS="-I$libs3_dir/include"
+          AC_SUBST(LIBS3_CFLAGS)
+          AC_MSG_RESULT($libs3_dir)
+        else
+          AC_MSG_ERROR([no libs3 found at $libs3_dir])
+        fi
+      else
+	AC_CHECK_LIB(s3, S3_deinitialize,
+        [
+            AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3])
+	    LIBS3_LDFLAGS='-ls3'
+            AC_SUBST(LIBS3_LDFLAGS)
+        ],
+        [
+            AC_MSG_ERROR([no libs3 found])
+        ])
+      fi
+    fi
+    ],
+    [
+	AC_CHECK_LIB(s3, S3_deinitialize,
+        [
+            AC_DEFINE([HAVE_LIBS3], [1], [Have found libs3])
+	    LIBS3_LDFLAGS='-ls3'
+            AC_SUBST(LIBS3_LDFLAGS)
+        ],
+        [])
+    ])
+
 AC_CHECK_FUNCS([bzero gethostbyname gettimeofday inet_ntoa memset mkdir pow select socket strchr strdup strerror strrchr strstr strtol strtoul utime],[],[AC_MSG_ERROR([One of the required functions was not found])])
 AC_CHECK_FUNCS([daemon getloadavg],[],[])
 

includes/metatile.h

@@ -12,21 +12,22 @@ extern "C" {
 
 #define META_MAGIC "META"
 #define META_MAGIC_COMPRESSED "METZ"
-
-    struct entry {
-        int offset;
-        int size;
-    };
-
-    struct meta_layout {
-        char magic[4];
-        int count; // METATILE ^ 2
-        int x, y, z; // lowest x,y of this metatile, plus z
-        struct entry index[]; // count entries
-        // Followed by the tile data
-        // The index offsets are measured from the start of the file
-    };
 
+struct entry {
+    int offset;
+    int size;
+};
+
+struct meta_layout {
+    char magic[4];
+    int count; // METATILE ^ 2
+    int x, y, z; // lowest x,y of this metatile, plus z
+    struct entry index[]; // count entries
+    // Followed by the tile data
+    // The index offsets are measured from the start of the file
+};
+
+#define METATILE_HEADER_LEN (sizeof(struct meta_layout) + METATILE * METATILE * sizeof(struct entry))
 
 #ifdef __cplusplus
 }

includes/store_file.h

@@ -6,9 +6,8 @@ extern "C" {
 #endif
 
 #include "store.h"
-
-    struct storage_backend * init_storage_file(const char * tile_dir);
-    int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z);
+
+struct storage_backend * init_storage_file(const char * tile_dir);
 
 #ifdef __cplusplus
 }

includes/store_file_utils.h

@@ -27,6 +27,7 @@ int path_to_xyz(const char *tilepath, const char *path, char *xmlconfig, int *px
 /* New meta-tile storage functions */
 /* Returns the path to the meta-tile and the offset within the meta-tile */
 int xyz_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, int x, int y, int z);
+int xyzo_to_meta(char *path, size_t len, const char *tile_dir, const char *xmlconfig, const char *options, int x, int y, int z);
 #endif
 
 #ifdef __cplusplus

includes/store_s3.h

@@ -0,0 +1,16 @@
+#ifndef STORES3_H
+#define STORES3_H
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+#include "store.h"
+
+struct storage_backend* init_storage_s3(const char *connection_string);
+
+#ifdef __cplusplus
+}
+#endif
+#endif

mod_tile.conf

@@ -14,6 +14,7 @@ LoadModule tile_module modules/mod_tile.so
 # The file based storage uses a simple file path as its storage path ( /path/to/tiledir )
 # The RADOS based storage takes a location to the rados config file and a pool name ( rados://poolname/path/to/ceph.conf )
 # The memcached based storage currently has no configuration options and always connects to memcached on localhost ( memcached:// )
+# The S3 based storage takes an access key, bucket, and path ( s3://access_key_id:secret_access_key[@host]/bucket_name[/basepath] )
 #
 # The storage path can be overwritten on a style by style basis from the style TileConfigFile
     ModTileTileDir /var/lib/mod_tile

readme.txt

@@ -46,8 +46,8 @@ daemon to render (or re-render) the tile.
 resources on the server and how out of date they are.
 
 4) Use tile storage other than a plain posix file system.
-e.g it can store tiles in a ceph object store, or proxy them
-from another tile server.
+e.g it can store tiles in a ceph object store, an Amazon S3 bucket,
+or proxy them from another tile server.
 
 5) Tile expiry. It estimates when the tile is next
 likely to be rendered and adds the appropriate HTTP

src/daemon.c

@@ -430,8 +430,8 @@ int server_socket_init(renderd_config *sConfig) {
         addrI.sin6_addr = in6addr_any;
         addrI.sin6_port = htons(sConfig->ipport);
         if (bind(fd, (struct sockaddr *) &addrI, sizeof(addrI)) < 0) {
-            fprintf(stderr, "socket bind failed for: %s:%i\n",
-                    sConfig->iphostname, sConfig->ipport);
+            fprintf(stderr, "socket bind failed for: %s:%i: %s\n",
+                    sConfig->iphostname, sConfig->ipport, strerror(errno));
             close(fd);
             exit(3);
         }
@@ -453,7 +453,8 @@ int server_socket_init(renderd_config *sConfig) {
 
         old = umask(0); // Need daemon socket to be writeable by apache
         if (bind(fd, (struct sockaddr *) &addrU, sizeof(addrU)) < 0) {
-            fprintf(stderr, "socket bind failed for: %s\n", sConfig->socketname);
+            fprintf(stderr, "socket bind failed for: %s: %s\n",
+                    sConfig->socketname, strerror(errno));
             close(fd);
             exit(3);
         }
@@ -932,7 +933,7 @@ int main(int argc, char **argv)
     if (foreground) {
         fprintf(stderr, "Running in foreground mode...\n");
     } else {
-        if (daemon(0, 0) != 0) {
+        if (daemon(1, 1) != 0) {
             fprintf(stderr, "can't daemonize: %s\n", strerror(errno));
         }
         /* write pid file */