npm package: @gulpetl/vinyl-s3
Description: Use S3 as a source or destination of vinyl files.
Author: peterl@sbcotton.com
Latest version: 0.4.3

# @gulpetl/vinyl-s3

Use S3 as a source or destination of vinyl files. Forked from [vinyl-s3](https://github.com/izaakschroeder/vinyl-s3) to add support for Node-RED as part of the [gulpetl](https://gulpetl.com/) project

![build status](http://img.shields.io/travis/izaakschroeder/vinyl-s3/master.svg?style=flat)
![coverage](http://img.shields.io/coveralls/izaakschroeder/vinyl-s3/master.svg?style=flat)
![license](http://img.shields.io/npm/l/vinyl-s3.svg?style=flat)
![version](http://img.shields.io/npm/v/vinyl-s3.svg?style=flat)
![downloads](http://img.shields.io/npm/dm/vinyl-s3.svg?style=flat)

Features:
 * Source with multi-globbing support,
 * Use either streaming or buffering,
 * Upload or download files,
 * Pass custom options to S3,
 * Smart `Content-Type` and `Content-Encoding` detection,
 * Works great with gulp.

## Usage

```javascript
var gulp = require('gulp'),
	s3 = require('vinyl-s3');

// Upload files to S3
gulp.task('upload', function() {
	return gulp.src('data/*.jpg', { buffer: false })
		.pipe(s3.dest('s3://my-bucket/prefix'));
});

// Download files from S3
gulp.task('download', function() {
	return s3.src('s3://my-bucket/prefix/*.jpg', { buffer: false })
		.pipe(gulp.dest('data'));
});

// Just print a list of files
var through2 = require('through2');
gulp.task('meta', function() {
	return s3.src('s3://my-bucket/foo/**/*.jpg', { read: false })
		.pipe(through2.obj(function(file, _, callback) {
			console.log(file.path);
			callback();
		}));
});
```

When working with large files you may find it useful to use streaming mode instead of buffering mode. You can enable this in the `src()` family of functions by setting `{ buffer: false }`. The default mode is to use buffering as is the same with `fs.src`.

### src

See [getObject] for a list of supported options.

```javascript
// Specify custom attributes via S3 URL.
s3.src('s3://bucket/key/*?IfModifiedSince=123456789')
    .pipe(fs.dest('downloads'));
```

```javascript
// Specify custom attributes by passing in an AWS options object.
s3.src({
    Bucket: 'bucket',
    Key: 'key/*',
    IfModifiedSince: Date.now()
}).pipe(fs.dest('downloads'));
```

```javascript
// Use multiple source buckets and patterns.
s3.src(['s3://bucket1/*.jpg', 's3://bucket1/*.png', 's3://bucket2/*.gif'])
    .pipe(fs.dest('downloads'));
```

### dest

See [putObject] and [upload] for a list of supported options. There is limited support for automatically detecting the correct `Content-Type` and correct `Content-Encoding`. Parallel uploads are supported by passing `{ queueSize: n }` as a second parameter.

```javascript
// Specify custom attributes via S3 URL.
fs.src('files/*.jpg')
    .pipe(s3.dest('s3://bucket/foo?ContentType=image/jpeg'));
```

```javascript
// Specify custom attributes by passing in an AWS options object.
fs.src('files/*.jpg')
    .pipe(s3.dest({
        Bucket: 'bucket',
        Key: 'foo',
        ContentType: 'image/jpeg'
    }));
```

```javascript
// Specify custom attributes per file.
fs.src('files/*.jpg')
    .pipe(through2.obj(function(file, enc, next) {
        // There are some non-standard properties on the file object that
        // are used to generate certain AWS options.
        file.contentType = 'image/jpeg';
        file.contentEncoding = 'gzip';

        // Setting the awsOptions property on a file causes the object to be
        // included in the command sent to S3. These options override any
        // previously set value.
        file.awsOptions = {
            ACL: 'private',
            CacheControl: 'max-age=1296000',
            ContentType: 'image/jpeg',
            Metadata: {
                color: 'red'
            }
        };
        this.push(file);
        next();
    }))
    .pipe(s3.dest('s3://bucket/foo'));
```

[getObject]: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#getObject-property
[putObject]: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
[upload]: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property


### Node-RED ###
[Node-RED](https://nodered.org/) is a low-code, visual programming environment for event-driven applications. vinyl-s3 is available there as part of the [gulp-etl](http://gulpetl.com) project; to install it to Node-RED under Manage Palette search for `vinyl-s3`

#### Demo flow ####

![Demo flow](./resources/demoflow.png)


Copy this flow for import in Node-RED under `Import`:

``` json
[{"id":"ea8a01833caca064","type":"inject","z":"900570325b249093","name":"click to start","props":[{"p":"payload"},{"p":"topic","vt":"str"}],"repeat":"","crontab":"","once":false,"onceDelay":0.1,"topic":"",
"payload":"","payloadType":"date","x":850,"y":40,"wires":[["581a22165aef5763"]]},{"id":"34e02d4be11bc045","type":"gulp.dest","z":"900570325b249093","name":"","path":"data","x":1210,"y":160,"wires":[["9d4b2a68ebec230e"]]},
{"id":"9d4b2a68ebec230e","type":"debug","z":"900570325b249093","name":"debug","active":true,"tosidebar":true,"console":false,"tostatus":false,"complete":"payload","targetType":"msg","statusVal":"","statusType":"auto",
"x":1290,"y":80,"wires":[]},{"id":"f03409d378908dad","type":"s3.src","z":"900570325b249093","name":"","path":"s3://my-bucket/prefix/*.jpg","x":970,"y":160,"wires":[["34e02d4be11bc045"]]},{"id":"e3f3197743972a9b","type":"comment",
"z":"900570325b249093","name":"Download files from S3","info":"","x":1100,"y":60,"wires":[]},{"id":"581a22165aef5763","type":"template","z":"900570325b249093","name":"Set config","field":"config","fieldType":"msg",
"format":"json","syntax":"plain","template":"{\n    \"buffer\": false,\n    \"s3\": {\n        \"accessKeyId\": \"AWS_ACCESS_ID\",\n        \"secretAccessKey\": \"AWS_ACCESS_KEY\",\n        \"region\": \"us-west-2\"\n    }\n}",
"output":"json","x":880,"y":100,"wires":[["f03409d378908dad"]]}]
```