From 81fe341975af4b93eef4bd76824c88cc95a9d21d Mon Sep 17 00:00:00 2001 From: Coen Hyde Date: Fri, 9 May 2014 13:53:04 -0700 Subject: [PATCH 1/6] implemented http callback notification for when job is complete --- lib/client.js | 26 ++++---------------- lib/worker.js | 68 +++++++++++++++++++++++++++++++++++++-------------- package.json | 3 ++- 3 files changed, 56 insertions(+), 41 deletions(-) diff --git a/lib/client.js b/lib/client.js index 41f0586..ee5cb4a 100644 --- a/lib/client.js +++ b/lib/client.js @@ -41,24 +41,11 @@ Client.prototype.upload = function(source, destination, callback) { * @param function callback The callback function. Optional. */ Client.prototype.thumbnail = function(originalImagePath, thumbnailDescriptions, callback) { - /** - job = { - "resources": [ - "/foo/awesome.jpg" - ], - "prefix": "/foo/awesome", - "descriptions": [{ - "suffix": "small", - "width": 64, - "height": 64 - }], - } - */ - this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify({ + this.job({ resources: [ originalImagePath ], prefix: originalImagePath.split('.').slice(0, -1).join('.'), descriptions: thumbnailDescriptions - })}, function (err, result) { + }, function (err, result) { if (callback) callback(err, result); }); }; @@ -71,7 +58,7 @@ Client.prototype.thumbnail = function(originalImagePath, thumbnailDescriptions, * @param array taskDescriptions Task meta information, see README.md. * @param function callback The callback function. Optional. */ -Client.prototype.task = function(resources, prefix, taskDescriptions, callback) { +Client.prototype.job = function(job, callback) { /** job = { "resources": [ @@ -85,13 +72,10 @@ Client.prototype.task = function(resources, prefix, taskDescriptions, callback) "width": 64, "height": 64 }], + "notify": "http://url.to.ping/when/job/is/complete" } */ - this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify({ - resources: resources, - prefix: prefix, - descriptions: taskDescriptions - })}, function (err, result) { + this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify(job)}, function (err, result) { if (callback) callback(err, result); }); }; diff --git a/lib/worker.js b/lib/worker.js index 496c995..eb75a7c 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -5,6 +5,7 @@ var aws = require('aws-sdk'), Thumbnailer = require('./thumbnailer').Thumbnailer, Saver = require('./saver').Saver, fs = require('fs'), + request = require('request') async = require('async'); /** @@ -100,22 +101,28 @@ Worker.prototype._runJob = function(handle, job, callback) { */ var _this = this; - async.mapLimit(job.resources, 5, function(resource, done) { - _this._downloadFromS3(resource, done); - }, function(err, localPaths) { - if (err) { - console.log(err); - callback(); - return; - } - _this._createThumbnails(localPaths, job, function(err) { - async.forEach(localPaths, fs.unlink, function(err) { - if (!err) { - _this._deleteJob(handle); - } - callback(); + async.waterfall([ + function(done) { + async.mapLimit(job.resources, 5, function(resource, done) { + _this._downloadFromS3(resource, done); + }, done); + }, + function(localPaths, done) { + _this._createThumbnails(localPaths, job, function(err, uploadedFiles) { + async.forEach(localPaths, fs.unlink, function(err) { + done(err, uploadedFiles); + }); }); - }); + }, + function(uploadedFiles, done) { + job.output = uploadedFiles; + _this._notify(job, done); + } + ], function(err) { + if (!err) { + _this._deleteJob(handle); + } + callback(); }); }; @@ -164,7 +171,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) { } else { _this._saveThumbnailToS3(convertedImagePath, remoteImagePath, function(err) { if (err) console.log(err); - done(); + done(null, remoteImagePath); }); } @@ -174,9 +181,7 @@ Worker.prototype._createThumbnails = function(localPaths, job, callback) { }); // perform thumbnailing in parallel. - async.parallel(work, function(err, results) { - callback(err); - }); + async.parallel(work, callback); }; @@ -221,4 +226,29 @@ Worker.prototype._deleteJob = function(handle) { }); }; +/** + * Call notification url + * + * @param string handle The SQS message handle + */ + +Worker.prototype._notify = function(job, cb) { + if (!job.notify) { + return cb(); + } + + var options = { + method: "POST", + url: job.notify, + json: true, + body: job + } + request.post(options, function(err) { + if (!err) { + console.log('notified '+job.notify); + } + return cb(); + }); +} + exports.Worker = Worker; diff --git a/package.json b/package.json index 635d802..c562582 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "tmp": "~0.0.16", "optimist": "~0.3.4", "async": "~0.2.7", - "sprintf-js": "0.0.7" + "sprintf-js": "0.0.7", + "request": "~2.34.0" }, "devDependencies": { "mocha": "~1.7.4", From d0dad54b029f1cc080321b4e60149375b2bb1afe Mon Sep 17 00:00:00 2001 From: bcoe Date: Thu, 15 May 2014 12:14:25 -0400 Subject: [PATCH 2/6] spacing fix. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58e95ba..334da08 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ client.upload('/tmp/awesome.jpg', destination, function(err) { * **originalImagePaths:** `string` or `array`, path to image or images that thumbnailing should be applied to. * **thumbnailDescriptions:** `array` describing the thumbnails that should be created. * **opts:** additional thumbnailing options. - * **notify:** webhook to notify when thumbnailing is complete. + * **notify:** webhook to notify when thumbnailing is complete. * **prefix:** prefix for thumbnails created (defaults to original filename). Thumbnail Descriptions From 51a8005ae9c677d3d4281a41741d705f28e46c55 Mon Sep 17 00:00:00 2001 From: bcoe Date: Thu, 15 May 2014 12:29:41 -0400 Subject: [PATCH 3/6] fixing README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 334da08..8798015 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The thumbd server: * HTTP resources are prefixed with `http://` or `https://`. * S3 resources are a path to the image in the S3 bucket indicated by the `BUCKET` environment variable. * Uses ImageMagick to perform a set of transformations on the image. -* uploads the thumbnails created back to S3, with the following naming convention: `[original filename excluding extension]\_[thumbnail suffix].[thumbnail format]` +* uploads the thumbnails created back to S3, with the following naming convention: `[original filename excluding extension]_[thumbnail suffix].[thumbnail format]` Assume that the following thumbnail job was received over SQS: From b795682363581c16ac0dd2f1b20a8f74a83b311a Mon Sep 17 00:00:00 2001 From: bcoe Date: Thu, 15 May 2014 12:34:15 -0400 Subject: [PATCH 4/6] fixed up license. --- LICENSE.txt | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 0f40555..44c1c26 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2012 Attachments.me +Copyright (c) 2012 Contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.md b/README.md index 8798015..993c26e 100644 --- a/README.md +++ b/README.md @@ -125,7 +125,7 @@ _description_ accepts the following keys: * **bounded (default):** maintain aspect ratio, don't place image on matte. * **matted:** maintain aspect ratio, places image on _width x height_ matte. * **fill:** both resizes and zooms into an image, filling the specified dimensions. - * **strict:** resizes the image, filling the specified dimensions changing the aspect ratio + * **strict:** resizes the image, filling the specified dimensions changing the aspect ratio * **manual:** allows for a custom convert command to be passed in: * `%(command)s -border 0 %(localPaths[0])s %(convertedPath)s` * **quality:** the quality of the thumbnail, in percent. e.g. `90`. @@ -200,4 +200,4 @@ thumbd is a rough first pass at creating an efficient, easy to deploy, thumbnail Copyright --------- -Copyright (c) 2012 Attachments.me. See LICENSE.txt for further details. +Copyright (c) 2014 Contributors, See LICENSE.txt for further details. From 9827348fa488daa7b234fe4bf8fb65657d2fd5eb Mon Sep 17 00:00:00 2001 From: bcoe Date: Thu, 15 May 2014 20:08:28 -0400 Subject: [PATCH 5/6] fixed a couple minor bugs that working on a plane brought up, added projects to README.md, made it so the thumbnail command accepts arbitrary keys. --- README.md | 12 +++++++----- lib/client.js | 11 ++++------- lib/grabber.js | 8 +++++--- lib/thumbnailer.js | 3 ++- package.json | 2 +- test/test-client.js | 17 +++++++++++++++++ 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 993c26e..4a8680c 100644 --- a/README.md +++ b/README.md @@ -189,13 +189,15 @@ At Attachments.me, thumbd thumbnailed tens of thousands of images a day. There a * we use long-polling to reduce the latency time before a message is read. * in production, thumbd runs on Node 0.8.x. It has not been thoroughly tested with Streams 2. -The Future ----------- +Projects Using Thumbd +-------------------- -thumbd is a rough first pass at creating an efficient, easy to deploy, thumbnailing pipeline. Please be liberal with your feature-requests, patches, and feedback: +**If you build something cool using thumbd let me know, I will list it here.** -* **If you create a client in a language other than JavaScript, let me know.** -* **If you build something cool using thumbd let me know, I will list it here.** +* **[Popbasic](https://popbasic.com)**: designs limited edition, high quality clothing. +* **[ineffable](https://github.com/taeram/ineffable/):** A minimalist photo album powered by Flask and React. +* **[s3-gif](https://github.com/taeram/s3-gif):** Host your GIF collection on Heroku + Amazon S3. +* **attachments.me**: created a searchable, visual, index of all of your email attachments (sadly defunct). Copyright --------- diff --git a/lib/client.js b/lib/client.js index e0c3b34..29e1835 100644 --- a/lib/client.js +++ b/lib/client.js @@ -76,15 +76,12 @@ Client.prototype.thumbnail = function(originalImagePaths, thumbnailDescriptions, // override defaults with opts. opts = _.extend({ - prefix: originalImagePaths[0].split('.').slice(0, -1).join('.') - }, opts); - - this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify({ + prefix: originalImagePaths[0].split('.').slice(0, -1).join('.'), resources: originalImagePaths, - prefix: opts.prefix, - notify: opts.notify, descriptions: thumbnailDescriptions - })}, function (err, result) { + }, opts); + + this.sqs.sendMessage({QueueUrl: config.get('sqsQueueUrl'), MessageBody: JSON.stringify(opts)}, function (err, result) { if (callback) callback(err, result); }); }; diff --git a/lib/grabber.js b/lib/grabber.js index 2db87e2..4cd09a7 100644 --- a/lib/grabber.js +++ b/lib/grabber.js @@ -105,11 +105,13 @@ Grabber.prototype.getFileS3 = function(remoteImagePath, localImagePath, stream, var _this = this, req = this.s3.getFile(remoteImagePath, function(err, res) { + // no response should count as an error. + if (!res) res = {statusCode: 503} + if (err || res.statusCode >= 400) { - err = 'error retrieving from S3 status ' + res.statusCode; stream.end(); - callback(err); - return; + console.log(err); + return callback(Error('error retrieving from S3 status ' + res.statusCode)); } res.pipe(stream); diff --git a/lib/thumbnailer.js b/lib/thumbnailer.js index 6993f6f..a16a62c 100644 --- a/lib/thumbnailer.js +++ b/lib/thumbnailer.js @@ -52,7 +52,8 @@ Thumbnailer.prototype.execute = function(description, localPaths, onComplete) { return; } - _this[_this._guessStrategy()](); + var strategy = _this._guessStrategy(); + if (strategy) _this[strategy](); }); }; diff --git a/package.json b/package.json index 0bae2f5..21c7cd5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "thumbd", - "version": "2.7.0", + "version": "2.8.0", "directories": { "lib": "./lib", "bin": "./bin", diff --git a/test/test-client.js b/test/test-client.js index f55aa87..43fc945 100644 --- a/test/test-client.js +++ b/test/test-client.js @@ -36,6 +36,23 @@ describe('thumbnail', function() { client.thumbnail('/foo/bar.jpg', [], {prefix: '/banana'}); }); + it("should allow arbitrary additional parameters to be set in opts", function(done) { + var client = new Client({ + sqs: { + sendMessage: function(sqsObject) { + var obj = JSON.parse( + sqsObject.MessageBody + ) + assert.equal(obj.foo, 'bar'); + done(); + }, + endpoint: {} // adhere to SQS contract. + }, + }); + + client.thumbnail('/foo/bar.jpg', [], {foo: 'bar'}); + }); + it('should execute callback when it is the third parameter', function(done) { var client = new Client({ sqs: { From 56a781c2d5bb34cfb9f76468a635588cd10ce2bb Mon Sep 17 00:00:00 2001 From: bcoe Date: Thu, 15 May 2014 20:11:36 -0400 Subject: [PATCH 6/6] missing comma. --- lib/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/worker.js b/lib/worker.js index f51fc6c..56955d1 100644 --- a/lib/worker.js +++ b/lib/worker.js @@ -5,7 +5,7 @@ var aws = require('aws-sdk'), Thumbnailer = require('./thumbnailer').Thumbnailer, Saver = require('./saver').Saver, fs = require('fs'), - request = require('request') + request = require('request'), async = require('async'); /**