Merge pull request #37 from bcoe/notification-merge

Merged #35 with Master, with a few minor Tweaks

LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2012 Attachments.me
+Copyright (c) 2012 Contributors
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

README.md

@@ -39,7 +39,7 @@ The thumbd server:
 	* HTTP resources are prefixed with `http://` or `https://`.
 	* S3 resources are a path to the image in the S3 bucket indicated by the `BUCKET` environment variable.
 * Uses ImageMagick to perform a set of transformations on the image.
-* uploads the thumbnails created back to S3, with the following naming convention: `[original filename excluding extension]\_[thumbnail suffix].[thumbnail format]`
+* uploads the thumbnails created back to S3, with the following naming convention: `[original filename excluding extension]_[thumbnail suffix].[thumbnail format]`
 
 Assume that the following thumbnail job was received over SQS:
 
@@ -94,10 +94,21 @@ var destination = '/example/awesome.jpg';
 
 client.upload('/tmp/awesome.jpg', destination, function(err) {
 	if (err) throw err;
-	client.thumbnail(destination, [{suffix: 'small', width: 100, height: 100, background: 'red', strategy: 'matted'}]);
+	client.thumbnail(originalImagePaths, [{suffix: 'small', width: 100, height: 100, background: 'red', strategy: 'matted'}], {
+		notify: 'https://callback.example.com', // optional web-hook when processing is done.
+		prefix: 'foobar' // optional prefix for thumbnails created.
+	});
 });
 ```
 
+**Thumbnailing options:**
+
+* **originalImagePaths:** `string` or `array`, path to image or images that thumbnailing should be applied to.
+* **thumbnailDescriptions:** `array` describing the thumbnails that should be created.
+* **opts:** additional thumbnailing options.
+	* **notify:** webhook to notify when thumbnailing is complete.
+	* **prefix:** prefix for thumbnails created (defaults to original filename).
+
 Thumbnail Descriptions
 ----------------------
 
@@ -114,7 +125,7 @@ _description_ accepts the following keys:
 	* **bounded (default):** maintain aspect ratio, don't place image on matte.
 	* **matted:** maintain aspect ratio, places image on _width x height_ matte.
 	* **fill:** both resizes and zooms into an image, filling the specified dimensions.
-  * **strict:** resizes the image, filling the specified dimensions changing the aspect ratio
+	* **strict:** resizes the image, filling the specified dimensions changing the aspect ratio
 	* **manual:** allows for a custom convert command to be passed in:
 	  * `%(command)s -border 0 %(localPaths[0])s %(convertedPath)s`
 * **quality:** the quality of the thumbnail, in percent. e.g. `90`.
@@ -137,7 +148,7 @@ thumbd thumbnail --remote_image=<path to image s3 or http> --descriptions=<path
 * **remote_image** indicates the S3 object to perform the thumbnailing operations on.
 * **thumbnail_descriptions** the path to a JSON file describing the dimensions of the thumbnails that should be created (see _example.json_ in the _data_ directory).
 
-Advanced Options
+Advanced Tips and Tricks
 ----------------
 
 * **Creating a Mosaic:** Rather than performing an operation on a single S3 resource, you can perform an operation on a set
@@ -178,15 +189,17 @@ At Attachments.me, thumbd thumbnailed tens of thousands of images a day. There a
 	* we use long-polling to reduce the latency time before a message is read.
 * in production, thumbd runs on Node 0.8.x. It has not been thoroughly tested with Streams 2.
 
-The Future
-----------
+Projects Using Thumbd
+--------------------
 
-thumbd is a rough first pass at creating an efficient, easy to deploy, thumbnailing pipeline. Please be liberal with your feature-requests, patches, and feedback:
+**If you build something cool using thumbd let me know, I will list it here.**
 
-* **If you create a client in a language other than JavaScript, let me know.**
-* **If you build something cool using thumbd let me know, I will list it here.**
+* **[Popbasic](https://popbasic.com)**: designs limited edition, high quality clothing.
+* **[ineffable](https://github.com/taeram/ineffable/):** A minimalist photo album powered by Flask and React.
+* **[s3-gif](https://github.com/taeram/s3-gif):** Host your GIF collection on Heroku + Amazon S3.
+* **attachments.me**: created a searchable, visual, index of all of your email attachments (sadly defunct).
 
 Copyright
 ---------
 
-Copyright (c) 2012 Attachments.me. See LICENSE.txt for further details.
+Copyright (c) 2014 Contributors, See LICENSE.txt for further details.

lib/client.js

@@ -76,14 +76,12 @@ Client.prototype.thumbnail = function(originalImagePaths, thumbnailDescriptions,
 
 	// override defaults with opts.
 	opts = _.extend({
-		prefix: originalImagePaths[0].split('.').slice(0, -1).join('.')
-	}, opts);
-
-	this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify({
+		prefix: originalImagePaths[0].split('.').slice(0, -1).join('.'),
 		resources: originalImagePaths,
-		prefix: opts.prefix,
 		descriptions: thumbnailDescriptions
-	})}, function (err, result) {
+	}, opts);
+
+	this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify(opts)}, function (err, result) {
 		if (callback) callback(err, result);
 	});
 };

lib/grabber.js

@@ -105,11 +105,13 @@ Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream,
 	var _this = this,
 		req = this.s3.getFile(remoteImagePath, function(err, res) {
 
+			// no response should count as an error.
+			if (!res) res = {statusCode: 503}
+
 			if (err || res.statusCode >= 400) {
-				err = 'error retrieving from S3 status ' + res.statusCode;
 				stream.end();
-				callback(err);
-				return;
+				console.log(err);
+				return callback(Error('error retrieving from S3 status ' + res.statusCode));
 			}
 
 			res.pipe(stream);

lib/thumbnailer.js

@@ -52,7 +52,8 @@ Thumbnailer.prototype.execute = function(description, localPaths, onComplete) {
 			return;
 		}
 
-		_this[_this._guessStrategy()]();
+		var strategy = _this._guessStrategy();
+		if (strategy) _this[strategy]();
 	});
 };
 

lib/worker.js

@@ -5,6 +5,7 @@ var aws = require('aws-sdk'),
 	Thumbnailer = require('./thumbnailer').Thumbnailer,
 	Saver = require('./saver').Saver,
 	fs = require('fs'),
+	request = require('request'),
 	async = require('async');
 
 /**
@@ -107,25 +108,28 @@ Worker.prototype._runJob = function(handle, job, callback) {
 
 	var _this = this;
 
-	async.mapLimit(job.resources, 5, function(resource, done) {
-		_this._downloadFromS3(resource, done);
-	}, function(err, localPaths) {
-		if (err) {
-			// We don't delete the job,
-			// in hopes that on the next retry
-			// it will succeed.
-			console.log(err);
-			callback();
-			return;
-		}
-		_this._createThumbnails(localPaths, job, function(err) {
-			async.forEach(localPaths, fs.unlink, function(err) {
-				if (!err) {
-					_this._deleteJob(handle);
-				}
-				callback();
+	async.waterfall([
+		function(done) {
+			async.mapLimit(job.resources, 5, function(resource, done) {
+				_this._downloadFromS3(resource, done);
+			}, done);
+		},
+		function(localPaths, done) {
+			_this._createThumbnails(localPaths, job, function(err, uploadedFiles) {
+				async.forEach(localPaths, fs.unlink, function(err) {
+					done(err, uploadedFiles);
+				});
 			});
-		});
+		},
+		function(uploadedFiles, done) {
+			job.output = uploadedFiles;
+			_this._notify(job, done);
+		}
+	], function(err) {
+		if (!err) {
+			_this._deleteJob(handle);
+		}
+		callback();
 	});
 };
 
@@ -174,7 +178,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
 				} else {
 					_this._saveThumbnailToS3(convertedImagePath, remoteImagePath, function(err) {
 						if (err) console.log(err);
-						done();
+						done(null, remoteImagePath);
 					});
 				}
 
@@ -184,10 +188,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) {
 	});
 
 	// perform thumbnailing in parallel.
-	async.parallel(work, function(err, results) {
-		callback(err);
-	});
-
+	async.parallel(work, callback);
 };
 
 /**
@@ -231,4 +232,27 @@ Worker.prototype._deleteJob = function(handle) {
 	});
 };
 
+/**
+ * Call notification url
+ *
+ * @param string job: the body of the SQS job.
+ */
+Worker.prototype._notify = function(job, cb) {
+  if (!job.notify) return cb();
+
+	var options = {
+		method: "POST",
+		url: job.notify,
+		json: true,
+		body: job
+	}
+
+	request.post(options, function(err) {
+		if (!err) {
+			console.log('notified:', job.notify);
+		}
+		return cb();
+	});
+}
+
 exports.Worker = Worker;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "thumbd",
-  "version": "2.7.0",
+  "version": "2.8.0",
   "directories": {
     "lib": "./lib",
     "bin": "./bin",
@@ -34,7 +34,8 @@
     "tmp": "~0.0.16",
     "optimist": "~0.3.4",
     "async": "~0.2.7",
-    "sprintf-js": "0.0.7"
+    "sprintf-js": "0.0.7",
+    "request": "~2.34.0"
   },
   "devDependencies": {
     "mocha": "~1.7.4",

test/test-client.js

@@ -36,6 +36,23 @@ describe('thumbnail', function() {
     client.thumbnail('/foo/bar.jpg', [], {prefix: '/banana'});
   });
 
+  it("should allow arbitrary additional parameters to be set in opts", function(done) {
+    var client = new Client({
+      sqs: {
+        sendMessage: function(sqsObject) {
+          var obj = JSON.parse(
+            sqsObject.MessageBody
+          )
+          assert.equal(obj.foo, 'bar');
+          done();
+        },
+        endpoint: {} // adhere to SQS contract.
+      },
+    });
+
+    client.thumbnail('/foo/bar.jpg', [], {foo: 'bar'});
+  });
+
   it('should execute callback when it is the third parameter', function(done) {
     var client = new Client({
       sqs: {