1
0
mirror of https://github.com/mgerb/mywebsite synced 2026-01-11 18:32:50 +00:00

Added file upload and dynamic blog post pages on index

This commit is contained in:
2015-10-20 14:31:22 -05:00
parent 184abc8ac1
commit 5f1fa1cf96
1420 changed files with 269139 additions and 30 deletions

34
node_modules/multer/CHANGELOG.md generated vendored Normal file
View File

@@ -0,0 +1,34 @@
# Change log
All notable changes to this project will be documented in this file.
This project adheres to [Semantic Versioning](http://semver.org/).
## 1.0.6 - 2015-10-03
- Bugfix: always report limit errors
## 1.0.5 - 2015-09-19
- Bugfix: drain the stream before considering request done
## 1.0.4 - 2015-09-19
- Bugfix: propagate all errors from busboy
## 1.0.3 - 2015-08-06
- Bugfix: ensure file order is correct
## 1.0.2 - 2015-08-06
- Bugfix: don't hang when hitting size limit
## 1.0.1 - 2015-07-20
- Bugfix: decrement pending writes on error
## 1.0.0 - 2015-07-18
- Introduce storage engines
- Specify expected fields
- Follow the W3C JSON form spec

17
node_modules/multer/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,17 @@
Copyright (c) 2014 Hage Yaapa <[http://www.hacksparrow.com](http://www.hacksparrow.com)>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

266
node_modules/multer/README.md generated vendored Normal file
View File

@@ -0,0 +1,266 @@
# Multer [![Build Status](https://travis-ci.org/expressjs/multer.svg?branch=master)](https://travis-ci.org/expressjs/multer) [![NPM version](https://badge.fury.io/js/multer.svg)](https://badge.fury.io/js/multer) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg?style=flat)](https://github.com/feross/standard)
Multer is a node.js middleware for handling `multipart/form-data`, which is primarily used for uploading files. It is written
on top of [busboy](https://github.com/mscdex/busboy) for maximum efficiency.
**NOTE**: Multer will not process any form which is not multipart (`multipart/form-data`).
## Installation
```sh
$ npm install --save multer
```
## Usage
Multer adds a `body` object and a `file` or `files` object to the `request` object. The `body` object contains the values of the text fields of the form, the `file` or `files` object contains the files uploaded via the form.
Basic usage example:
```javascript
var express = require('express')
var multer = require('multer')
var upload = multer({ dest: 'uploads/' })
var app = express()
app.post('/profile', upload.single('avatar'), function (req, res, next) {
// req.file is the `avatar` file
// req.body will hold the text fields, if there were any
})
app.post('/photos/upload', upload.array('photos', 12), function (req, res, next) {
// req.files is array of `photos` files
// req.body will contain the text fields, if there were any
})
var cpUpload = upload.fields([{ name: 'avatar', maxCount: 1 }, { name: 'gallery', maxCount: 8 }])
app.post('/cool-profile', cpUpload, function (req, res, next) {
// req.files is an object (String -> Array) where fieldname is the key, and the value is array of files
//
// e.g.
// req.files['avatar'][0] -> File
// req.files['gallery'] -> Array
//
// req.body will contain the text fields, if there were any
})
```
In case you need to handle a text-only multipart form, you can use any of the multer methods (`.single()`, `.array()`, `fields()`). Here is an example using `.array()`:
```javascript
var express = require('express')
var app = express()
var multer = require('multer')
var upload = multer()
app.post('/profile', upload.array(), function (req, res, next) {
// req.body contains the text fields
})
```
## API
### File information
Each file contains the following information:
Key | Description | Note
--- | --- | ---
`fieldname` | Field name specified in the form |
`originalname` | Name of the file on the user's computer |
`encoding` | Encoding type of the file |
`mimetype` | Mime type of the file |
`size` | Size of the file in bytes |
`destination` | The folder to which the file has been saved | `DiskStorage`
`filename` | The name of the file within the `destination` | `DiskStorage`
`path` | The full path to the uploaded file | `DiskStorage`
`buffer` | A `Buffer` of the entire file | `MemoryStorage`
### `multer(opts)`
Multer accepts an options object, the most basic of which is the `dest`
property, which tells Multer where to upload the files. In case you omit the
options object, the files will be kept in memory and never written to disk.
By default, Multer will rename the files so as to avoid naming conflicts. The
renaming function can be customized according to your needs.
The following are the options that can be passed to Multer.
Key | Description
--- | ---
`dest` or `storage` | Where to store the files
`fileFilter` | Function to control which files are accepted
`limits` | Limits of the uploaded data
In an average web app, only `dest` might be required, and configured as shown in
the following example.
```javascript
var upload = multer({ dest: 'uploads/' })
```
If you want more control over your uploads, you'll want to use the `storage`
option instead of `dest`. Multer ships with storage engines `DiskStorage`
and `MemoryStorage`; More engines are available from third parties.
#### `.single(fieldname)`
Accept a single file with the name `fieldname`. The single file will be stored
in `req.file`.
#### `.array(fieldname[, maxCount])`
Accept an array of files, all with the name `fieldname`. Optionally error out if
more than `maxCount` files are uploaded. The array of files will be stored in
`req.files`.
#### `.fields(fields)`
Accept a mix of files, specified by `fields`. An object with arrays of files
will be stored in `req.files`.
`fields` should be an array of objects with `name` and optionally a `maxCount`.
Example:
```javascript
[
{ name: 'avatar', maxCount: 1 },
{ name: 'gallery', maxCount: 8 }
]
```
### `storage`
#### `DiskStorage`
The disk storage engine gives you full control on storing files to disk.
```javascript
var storage = multer.diskStorage({
destination: function (req, file, cb) {
cb(null, '/tmp/my-uploads')
},
filename: function (req, file, cb) {
cb(null, file.fieldname + '-' + Date.now())
}
})
var upload = multer({ storage: storage })
```
There are two options available, `destination` and `filename`. They are both
functions that determine where the file should be stored.
`destination` is used to determine within which folder the uploaded files should
be stored. This can also be given as a `string` (e.g. `'/tmp/uploads'`). If no
`destination` is given, the operating system's default directory for temporary
files is used.
**Note:** You are responsible for creating the directory when providing
`destination` as a function. When passing a string, multer will make sure that
the directory is created for you.
`filename` is used to determine what the file should be named inside the folder.
If no `filename` is given, each file will be given a random name that doesn't
include any file extension.
**Note:** Multer will not append any file extension for you, your function
should return a filename complete with an file extension.
Each function gets passed both the request (`req`) and some information about
the file (`file`) to aid with the decision.
Note that `req.body` might not have been fully populated yet. It depends on the
order that the client transmits fields and files to the server.
#### `MemoryStorage`
The memory storage engine stores the files in memory as `Buffer` objects. It
doesn't have any options.
```javascript
var storage = multer.memoryStorage()
var upload = multer({ storage: storage })
```
When using memory storage, the file info will contain a field called
`buffer` that contains the entire file.
**WARNING**: Uploading very large files, or relatively small files in large
numbers very quickly, can cause your application to run out of memory when
memory storage is used.
### `limits`
An object specifying the size limits of the following optional properties. Multer passes this object into busboy directly, and the details of the properties can be found on [busboy's page](https://github.com/mscdex/busboy#busboy-methods).
The following integer values are available:
Key | Description | Default
--- | --- | ---
`fieldNameSize` | Max field name size | 100 bytes
`fieldSize` | Max field value size | 1MB
`fields` | Max number of non-file fields | Infinity
`fileSize` | For multipart forms, the max file size (in bytes) | Infinity
`files` | For multipart forms, the max number of file fields | Infinity
`parts` | For multipart forms, the max number of parts (fields + files) | Infinity
`headerPairs` | For multipart forms, the max number of header key=>value pairs to parse | 2000
Specifying the limits can help protect your site against denial of service (DoS) attacks.
### `fileFilter`
Set this to a function to control which files should be uploaded and which
should be skipped. The function should look like this:
```javascript
function fileFilter (req, file, cb) {
// The function should call `cb` with a boolean
// to indicate if the file should be accepted
// To reject this file pass `false`, like so:
cb(null, false)
// To accept the file pass `true`, like so:
cb(null, true)
// You can always pass an error if something goes wrong:
cb(new Error('I don\'t have a clue!'))
}
```
## Error handling
When encountering an error, multer will delegate the error to express. You can
display a nice error page using [the standard express way](http://expressjs.com/guide/error-handling.html).
If you want to catch errors specifically from multer, you can call the
middleware function by yourself.
```javascript
var upload = multer().single('avatar')
app.post('/profile', function (req, res) {
upload(req, res, function (err) {
if (err) {
// An error occurred when uploading
return
}
// Everything went fine
})
})
```
## Custom storage engine
See [the documentation here](/StorageEngine.md) if you want to build your own
storage engine.
## License
[MIT](LICENSE)

83
node_modules/multer/index.js generated vendored Normal file
View File

@@ -0,0 +1,83 @@
var makeError = require('./lib/make-error')
var makeMiddleware = require('./lib/make-middleware')
var diskStorage = require('./storage/disk')
var memoryStorage = require('./storage/memory')
function allowAll (req, file, cb) {
cb(null, true)
}
function Multer (options) {
if (options.storage) {
this.storage = options.storage
} else if (options.dest) {
this.storage = diskStorage({ destination: options.dest })
} else {
this.storage = memoryStorage()
}
this.limits = options.limits
this.fileFilter = options.fileFilter || allowAll
}
Multer.prototype._makeMiddleware = function (fields, fileStrategy) {
function setup () {
var fileFilter = this.fileFilter
var filesLeft = Object.create(null)
fields.forEach(function (field) {
if (typeof field.maxCount === 'number') {
filesLeft[field.name] = field.maxCount
} else {
filesLeft[field.name] = Infinity
}
})
function wrappedFileFilter (req, file, cb) {
if ((filesLeft[file.fieldname] || 0) <= 0) {
return cb(makeError('LIMIT_UNEXPECTED_FILE', file.fieldname))
}
filesLeft[file.fieldname] -= 1
fileFilter(req, file, cb)
}
return {
limits: this.limits,
storage: this.storage,
fileFilter: wrappedFileFilter,
fileStrategy: fileStrategy
}
}
return makeMiddleware(setup.bind(this))
}
Multer.prototype.single = function (name) {
return this._makeMiddleware([{ name: name, maxCount: 1 }], 'VALUE')
}
Multer.prototype.array = function (name, maxCount) {
return this._makeMiddleware([{ name: name, maxCount: maxCount }], 'ARRAY')
}
Multer.prototype.fields = function (fields) {
return this._makeMiddleware(fields, 'OBJECT')
}
function multer (options) {
if (options === undefined) {
return new Multer({})
}
if (typeof options === 'object' && options !== null) {
return new Multer(options)
}
throw new TypeError('Expected object for argument options')
}
module.exports = multer
module.exports.diskStorage = diskStorage
module.exports.memoryStorage = memoryStorage

28
node_modules/multer/lib/counter.js generated vendored Normal file
View File

@@ -0,0 +1,28 @@
var EventEmitter = require('events').EventEmitter
function Counter () {
EventEmitter.call(this)
this.value = 0
}
Counter.prototype = Object.create(EventEmitter.prototype)
Counter.prototype.increment = function increment () {
this.value++
}
Counter.prototype.decrement = function decrement () {
if (--this.value === 0) this.emit('zero')
}
Counter.prototype.isZero = function isZero () {
return (this.value === 0)
}
Counter.prototype.onceZero = function onceZero (fn) {
if (this.isZero()) return fn()
this.once('zero', fn)
}
module.exports = Counter

64
node_modules/multer/lib/file-appender.js generated vendored Normal file
View File

@@ -0,0 +1,64 @@
var objectAssign = require('object-assign')
function arrayRemove (arr, item) {
var idx = arr.indexOf(item)
if (~idx) arr.splice(idx, 1)
}
function FileAppender (strategy, req) {
this.strategy = strategy
this.req = req
switch (strategy) {
case 'VALUE': break
case 'ARRAY': req.files = []; break
case 'OBJECT': req.files = Object.create(null); break
default: throw new Error('Unknown file strategy: ' + strategy)
}
}
FileAppender.prototype.insertPlaceholder = function (file) {
var placeholder = {
fieldname: file.fieldname
}
switch (this.strategy) {
case 'VALUE': break
case 'ARRAY': this.req.files.push(placeholder); break
case 'OBJECT':
if (this.req.files[file.fieldname]) {
this.req.files[file.fieldname].push(placeholder)
} else {
this.req.files[file.fieldname] = [placeholder]
}
break
}
return placeholder
}
FileAppender.prototype.removePlaceholder = function (placeholder) {
switch (this.strategy) {
case 'VALUE': break
case 'ARRAY': arrayRemove(this.req.files, placeholder); break
case 'OBJECT':
if (this.req.files[placeholder.fieldname].length === 1) {
delete this.req.files[placeholder.fieldname]
} else {
arrayRemove(this.req.files[placeholder.fieldname], placeholder)
}
break
}
}
FileAppender.prototype.replacePlaceholder = function (placeholder, file) {
if (this.strategy === 'VALUE') {
this.req.file = file
return
}
delete placeholder.fieldname
objectAssign(placeholder, file)
}
module.exports = FileAppender

18
node_modules/multer/lib/make-error.js generated vendored Normal file
View File

@@ -0,0 +1,18 @@
var errorMessages = {
'LIMIT_PART_COUNT': 'Too many parts',
'LIMIT_FILE_SIZE': 'File too large',
'LIMIT_FILE_COUNT': 'Too many files',
'LIMIT_FIELD_KEY': 'Field name too long',
'LIMIT_FIELD_VALUE': 'Field value too long',
'LIMIT_FIELD_COUNT': 'Too many fields',
'LIMIT_UNEXPECTED_FILE': 'Unexpected field'
}
function makeError (code, optionalField) {
var err = new Error(errorMessages[code])
err.code = code
if (optionalField) err.field = optionalField
return err
}
module.exports = makeError

180
node_modules/multer/lib/make-middleware.js generated vendored Normal file
View File

@@ -0,0 +1,180 @@
var is = require('type-is')
var Busboy = require('busboy')
var extend = require('xtend')
var onFinished = require('on-finished')
var appendField = require('append-field')
var Counter = require('./counter')
var makeError = require('./make-error')
var FileAppender = require('./file-appender')
var removeUploadedFiles = require('./remove-uploaded-files')
function drainStream (stream) {
stream.on('readable', stream.read.bind(stream))
}
function makeMiddleware (setup) {
return function multerMiddleware (req, res, next) {
if (!is(req, ['multipart'])) return next()
var options = setup()
var limits = options.limits
var storage = options.storage
var fileFilter = options.fileFilter
var fileStrategy = options.fileStrategy
req.body = Object.create(null)
var busboy
try {
busboy = new Busboy({ headers: req.headers, limits: limits })
} catch (err) {
return next(err)
}
var appender = new FileAppender(fileStrategy, req)
var isDone = false
var readFinished = false
var errorOccured = false
var pendingWrites = new Counter()
var uploadedFiles = []
function done (err) {
if (isDone) return
isDone = true
req.unpipe(busboy)
drainStream(req)
busboy.removeAllListeners()
onFinished(req, function () { next(err) })
}
function indicateDone () {
if (readFinished && pendingWrites.isZero() && !errorOccured) done()
}
function abortWithError (uploadError) {
errorOccured = true
pendingWrites.onceZero(function () {
function remove (file, cb) {
storage._removeFile(req, file, cb)
}
removeUploadedFiles(uploadedFiles, remove, function (err, storageErrors) {
if (err) return done(err)
uploadError.storageErrors = storageErrors
done(uploadError)
})
})
}
function abortWithCode (code, optionalField) {
abortWithError(makeError(code, optionalField))
}
// handle text field data
busboy.on('field', function (fieldname, value, fieldnameTruncated, valueTruncated) {
if (fieldnameTruncated) return abortWithCode('LIMIT_FIELD_KEY')
if (valueTruncated) return abortWithCode('LIMIT_FIELD_VALUE', fieldname)
// Work around bug in Busboy (https://github.com/mscdex/busboy/issues/6)
if (limits && limits.hasOwnProperty('fieldNameSize')) {
if (fieldname.length > limits.fieldNameSize) return abortWithCode('LIMIT_FIELD_KEY')
}
appendField(req.body, fieldname, value)
})
// handle files
busboy.on('file', function (fieldname, fileStream, filename, encoding, mimetype) {
// don't attach to the files object, if there is no file
if (!filename) return fileStream.resume()
// Work around bug in Busboy (https://github.com/mscdex/busboy/issues/6)
if (limits && limits.hasOwnProperty('fieldNameSize')) {
if (fieldname.length > limits.fieldNameSize) return abortWithCode('LIMIT_FIELD_KEY')
}
var file = {
fieldname: fieldname,
originalname: filename,
encoding: encoding,
mimetype: mimetype
}
var placeholder = appender.insertPlaceholder(file)
fileFilter(req, file, function (err, includeFile) {
if (err) {
appender.removePlaceholder(placeholder)
return abortWithError(err)
}
if (!includeFile) {
appender.removePlaceholder(placeholder)
return fileStream.resume()
}
var aborting = false
pendingWrites.increment()
Object.defineProperty(file, 'stream', {
configurable: true,
enumerable: false,
value: fileStream
})
fileStream.on('error', function (err) {
pendingWrites.decrement()
abortWithError(err)
})
fileStream.on('limit', function () {
aborting = true
abortWithCode('LIMIT_FILE_SIZE', fieldname)
})
storage._handleFile(req, file, function (err, info) {
if (aborting) {
appender.removePlaceholder(placeholder)
uploadedFiles.push(extend(file, info))
return pendingWrites.decrement()
}
if (err) {
appender.removePlaceholder(placeholder)
pendingWrites.decrement()
return abortWithError(err)
}
var fileInfo = extend(file, info)
appender.replacePlaceholder(placeholder, fileInfo)
uploadedFiles.push(fileInfo)
pendingWrites.decrement()
indicateDone()
})
})
})
busboy.on('error', function (err) { abortWithError(err) })
busboy.on('partsLimit', function () { abortWithCode('LIMIT_PART_COUNT') })
busboy.on('filesLimit', function () { abortWithCode('LIMIT_FILE_COUNT') })
busboy.on('fieldsLimit', function () { abortWithCode('LIMIT_FIELD_COUNT') })
busboy.on('finish', function () {
readFinished = true
indicateDone()
})
req.pipe(busboy)
}
}
module.exports = makeMiddleware

28
node_modules/multer/lib/remove-uploaded-files.js generated vendored Normal file
View File

@@ -0,0 +1,28 @@
function removeUploadedFiles (uploadedFiles, remove, cb) {
var length = uploadedFiles.length
var errors = []
if (length === 0) return cb(null, errors)
function handleFile (idx) {
var file = uploadedFiles[idx]
remove(file, function (err) {
if (err) {
err.file = file
err.field = file.fieldname
errors.push(err)
}
if (idx < length - 1) {
handleFile(idx + 1)
} else {
cb(null, errors)
}
})
}
handleFile(0)
}
module.exports = removeUploadedFiles

1
node_modules/multer/node_modules/.bin/mkdirp generated vendored Symbolic link
View File

@@ -0,0 +1 @@
../mkdirp/bin/cmd.js

View File

@@ -0,0 +1 @@
node_modules/

21
node_modules/multer/node_modules/append-field/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2015 Linus Unnebäck
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -0,0 +1,44 @@
# `append-field`
A [W3C HTML JSON forms spec](http://www.w3.org/TR/html-json-forms/) compliant
field appender (for lack of a better name). Useful for people implementing
`application/x-www-form-urlencoded` and `multipart/form-data` parsers.
It works best on objects created with `Object.create(null)`. Otherwise it might
conflict with variables from the prototype (e.g. `hasOwnProperty`).
## Installation
```sh
npm install --save append-field
```
## Usage
```javascript
var appendField = require('append-field')
var obj = Object.create(null)
appendField(obj, 'pets[0][species]', 'Dahut')
appendField(obj, 'pets[0][name]', 'Hypatia')
appendField(obj, 'pets[1][species]', 'Felis Stultus')
appendField(obj, 'pets[1][name]', 'Billie')
console.log(obj)
```
```text
{ pets:
[ { species: 'Dahut', name: 'Hypatia' },
{ species: 'Felis Stultus', name: 'Billie' } ] }
```
## API
### `appendField(store, key, value)`
Adds the field named `key` with the value `value` to the object `store`.
## License
MIT

12
node_modules/multer/node_modules/append-field/index.js generated vendored Normal file
View File

@@ -0,0 +1,12 @@
var parsePath = require('./lib/parse-path')
var setValue = require('./lib/set-value')
function appendField (store, key, value) {
var steps = parsePath(key)
steps.reduce(function (context, step) {
return setValue(context, step, context[step.key], value)
}, store)
}
module.exports = appendField

View File

@@ -0,0 +1,53 @@
var reFirstKey = /^[^\[]*/
var reDigitPath = /^\[(\d+)\]/
var reNormalPath = /^\[([^\]]+)\]/
function parsePath (key) {
function failure () {
return [{ type: 'object', key: key, last: true }]
}
var firstKey = reFirstKey.exec(key)[0]
if (!firstKey) return failure()
var len = key.length
var pos = firstKey.length
var tail = { type: 'object', key: firstKey }
var steps = [tail]
while (pos < len) {
var m
if (key[pos] === '[' && key[pos + 1] === ']') {
pos += 2
tail.append = true
if (pos !== len) return failure()
continue
}
m = reDigitPath.exec(key.substring(pos))
if (m !== null) {
pos += m[0].length
tail.nextType = 'array'
tail = { type: 'array', key: parseInt(m[1], 10) }
steps.push(tail)
continue
}
m = reNormalPath.exec(key.substring(pos))
if (m !== null) {
pos += m[0].length
tail.nextType = 'object'
tail = { type: 'object', key: m[1] }
steps.push(tail)
continue
}
return failure()
}
tail.last = true
return steps
}
module.exports = parsePath

View File

@@ -0,0 +1,64 @@
function valueType (value) {
if (value === undefined) return 'undefined'
if (Array.isArray(value)) return 'array'
if (typeof value === 'object') return 'object'
return 'scalar'
}
function setLastValue (context, step, currentValue, entryValue) {
switch (valueType(currentValue)) {
case 'undefined':
if (step.append) {
context[step.key] = [entryValue]
} else {
context[step.key] = entryValue
}
break
case 'array':
context[step.key].push(entryValue)
break
case 'object':
return setLastValue(currentValue, { type: 'object', key: '', last: true }, currentValue[''], entryValue)
case 'scalar':
context[step.key] = [context[step.key], entryValue]
break
}
return context
}
function setValue (context, step, currentValue, entryValue) {
if (step.last) return setLastValue(context, step, currentValue, entryValue)
var obj
switch (valueType(currentValue)) {
case 'undefined':
if (step.nextType === 'array') {
context[step.key] = []
} else {
context[step.key] = Object.create(null)
}
return context[step.key]
case 'object':
return context[step.key]
case 'array':
if (step.nextType === 'array') {
return currentValue
}
obj = Object.create(null)
context[step.key] = obj
currentValue.forEach(function (item, i) {
if (item !== undefined) obj['' + i] = item
})
return obj
case 'scalar':
obj = Object.create(null)
obj[''] = currentValue
context[step.key] = obj
return obj
}
}
module.exports = setValue

View File

@@ -0,0 +1,50 @@
{
"name": "append-field",
"version": "0.1.0",
"license": "MIT",
"author": {
"name": "Linus Unnebäck",
"email": "linus@folkdatorn.se"
},
"main": "index.js",
"devDependencies": {
"mocha": "^2.2.4",
"standard": "^3.7.2",
"testdata-w3c-json-form": "^0.2.0"
},
"scripts": {
"test": "standard && mocha"
},
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/LinusU/node-append-field.git"
},
"gitHead": "b88c9cc6a37273956843422ca2d384bd55b44b08",
"description": "A [W3C HTML JSON forms spec](http://www.w3.org/TR/html-json-forms/) compliant field appender (for lack of a better name). Useful for people implementing `application/x-www-form-urlencoded` and `multipart/form-data` parsers.",
"bugs": {
"url": "https://github.com/LinusU/node-append-field/issues"
},
"homepage": "https://github.com/LinusU/node-append-field",
"_id": "append-field@0.1.0",
"_shasum": "6ddc58fa083c7bc545d3c5995b2830cc2366d44a",
"_from": "append-field@>=0.1.0 <0.2.0",
"_npmVersion": "2.7.5",
"_nodeVersion": "0.12.2",
"_npmUser": {
"name": "linusu",
"email": "linus@folkdatorn.se"
},
"maintainers": [
{
"name": "linusu",
"email": "linus@folkdatorn.se"
}
],
"dist": {
"shasum": "6ddc58fa083c7bc545d3c5995b2830cc2366d44a",
"tarball": "http://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz"
},
"directories": {},
"_resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz",
"readme": "ERROR: No README data found!"
}

View File

@@ -0,0 +1,19 @@
/* eslint-env mocha */
var assert = require('assert')
var appendField = require('../')
var testData = require('testdata-w3c-json-form')
describe('Append Field', function () {
for (var test of testData) {
it('handles ' + test.name, function () {
var store = Object.create(null)
for (var field of test.fields) {
appendField(store, field.key, field.value)
}
assert.deepEqual(store, test.expected)
})
}
})

19
node_modules/multer/node_modules/busboy/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,19 @@
Copyright Brian White. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

216
node_modules/multer/node_modules/busboy/README.md generated vendored Normal file
View File

@@ -0,0 +1,216 @@
Description
===========
A node.js module for parsing incoming HTML form data.
If you've found this module to be useful and wish to support it, you may do so by visiting this pledgie campaign:
<a href='https://pledgie.com/campaigns/28774'><img alt='Click here to support busboy' src='https://pledgie.com/campaigns/28774.png?skin_name=chrome' border='0'></a>
Requirements
============
* [node.js](http://nodejs.org/) -- v0.8.0 or newer
Install
=======
npm install busboy
Examples
========
* Parsing (multipart) with default options:
```javascript
var http = require('http'),
inspect = require('util').inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename + ', encoding: ' + encoding + ', mimetype: ' + mimetype);
file.on('data', function(data) {
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
});
busboy.on('finish', function() {
console.log('Done parsing form!');
res.writeHead(303, { Connection: 'close', Location: '/' });
res.end();
});
req.pipe(busboy);
} else if (req.method === 'GET') {
res.writeHead(200, { Connection: 'close' });
res.end('<html><head></head><body>\
<form method="POST" enctype="multipart/form-data">\
<input type="text" name="textfield"><br />\
<input type="file" name="filefield"><br />\
<input type="submit">\
</form>\
</body></html>');
}
}).listen(8000, function() {
console.log('Listening for requests');
});
// Example output, using http://nodejs.org/images/ryan-speaker.jpg as the file:
//
// Listening for requests
// File [filefield]: filename: ryan-speaker.jpg, encoding: binary
// File [filefield] got 11971 bytes
// Field [textfield]: value: 'testing! :-)'
// File [filefield] Finished
// Done parsing form!
```
* Save all incoming files to disk:
```javascript
var http = require('http'),
path = require('path'),
os = require('os');
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
var saveTo = path.join(os.tmpDir(), path.basename(fieldname));
file.pipe(fs.createWriteStream(saveTo));
});
busboy.on('finish', function() {
res.writeHead(200, { 'Connection': 'close' });
res.end("That's all folks!");
});
return req.pipe(busboy);
}
res.writeHead(404);
res.end();
}).listen(8000, function() {
console.log('Listening for requests');
});
```
* Parsing (urlencoded) with default options:
```javascript
var http = require('http'),
inspect = require('util').inspect;
var Busboy = require('busboy');
http.createServer(function(req, res) {
if (req.method === 'POST') {
var busboy = new Busboy({ headers: req.headers });
busboy.on('file', function(fieldname, file, filename, encoding, mimetype) {
console.log('File [' + fieldname + ']: filename: ' + filename);
file.on('data', function(data) {
console.log('File [' + fieldname + '] got ' + data.length + ' bytes');
});
file.on('end', function() {
console.log('File [' + fieldname + '] Finished');
});
});
busboy.on('field', function(fieldname, val, fieldnameTruncated, valTruncated) {
console.log('Field [' + fieldname + ']: value: ' + inspect(val));
});
busboy.on('finish', function() {
console.log('Done parsing form!');
res.writeHead(303, { Connection: 'close', Location: '/' });
res.end();
});
req.pipe(busboy);
} else if (req.method === 'GET') {
res.writeHead(200, { Connection: 'close' });
res.end('<html><head></head><body>\
<form method="POST">\
<input type="text" name="textfield"><br />\
<select name="selectfield">\
<option value="1">1</option>\
<option value="10">10</option>\
<option value="100">100</option>\
<option value="9001">9001</option>\
</select><br />\
<input type="checkbox" name="checkfield">Node.js rules!<br />\
<input type="submit">\
</form>\
</body></html>');
}
}).listen(8000, function() {
console.log('Listening for requests');
});
// Example output:
//
// Listening for requests
// Field [textfield]: value: 'testing! :-)'
// Field [selectfield]: value: '9001'
// Field [checkfield]: value: 'on'
// Done parsing form!
```
API
===
_Busboy_ is a _Writable_ stream
Busboy (special) events
-----------------------
* **file**(< _string_ >fieldname, < _ReadableStream_ >stream, < _string_ >filename, < _string_ >transferEncoding, < _string_ >mimeType) - Emitted for each new file form field found. `transferEncoding` contains the 'Content-Transfer-Encoding' value for the file stream. `mimeType` contains the 'Content-Type' value for the file stream.
* Note: if you listen for this event, you should always handle the `stream` no matter if you care about the file contents or not (e.g. you can simply just do `stream.resume();` if you want to discard the contents), otherwise the 'finish' event will never fire on the Busboy instance. However, if you don't care about **any** incoming files, you can simply not listen for the 'file' event at all and any/all files will be automatically and safely discarded (these discarded files do still count towards `files` and `parts` limits).
* If a configured file size limit was reached, `stream` will both have a boolean property `truncated` (best checked at the end of the stream) and emit a 'limit' event to notify you when this happens.
* **field**(< _string_ >fieldname, < _string_ >value, < _boolean_ >fieldnameTruncated, < _boolean_ >valueTruncated) - Emitted for each new non-file field found.
* **partsLimit**() - Emitted when specified `parts` limit has been reached. No more 'file' or 'field' events will be emitted.
* **filesLimit**() - Emitted when specified `files` limit has been reached. No more 'file' events will be emitted.
* **fieldsLimit**() - Emitted when specified `fields` limit has been reached. No more 'field' events will be emitted.
Busboy methods
--------------
* **(constructor)**(< _object_ >config) - Creates and returns a new Busboy instance with the following valid `config` settings:
* **headers** - _object_ - These are the HTTP headers of the incoming request, which are used by individual parsers.
* **highWaterMark** - _integer_ - highWaterMark to use for this Busboy instance (Default: WritableStream default).
* **fileHwm** - _integer_ - highWaterMark to use for file streams (Default: ReadableStream default).
* **defCharset** - _string_ - Default character set to use when one isn't defined (Default: 'utf8').
* **preservePath** - _boolean_ - If paths in the multipart 'filename' field shall be preserved. (Default: false).
* **limits** - _object_ - Various limits on incoming data. Valid properties are:
* **fieldNameSize** - _integer_ - Max field name size (in bytes) (Default: 100 bytes).
* **fieldSize** - _integer_ - Max field value size (in bytes) (Default: 1MB).
* **fields** - _integer_ - Max number of non-file fields (Default: Infinity).
* **fileSize** - _integer_ - For multipart forms, the max file size (in bytes) (Default: Infinity).
* **files** - _integer_ - For multipart forms, the max number of file fields (Default: Infinity).
* **parts** - _integer_ - For multipart forms, the max number of parts (fields + files) (Default: Infinity).
* **headerPairs** - _integer_ - For multipart forms, the max number of header key=>value pairs to parse **Default:** 2000 (same as node's http).

File diff suppressed because one or more lines are too long

File diff suppressed because it is too large Load Diff

86
node_modules/multer/node_modules/busboy/lib/main.js generated vendored Normal file
View File

@@ -0,0 +1,86 @@
var fs = require('fs'),
WritableStream = require('stream').Writable
|| require('readable-stream').Writable,
inherits = require('util').inherits;
var parseParams = require('./utils').parseParams;
function Busboy(opts) {
if (!(this instanceof Busboy))
return new Busboy(opts);
if (opts.highWaterMark !== undefined)
WritableStream.call(this, { highWaterMark: opts.highWaterMark });
else
WritableStream.call(this);
this._done = false;
this._parser = undefined;
this.opts = opts;
if (opts.headers && typeof opts.headers['content-type'] === 'string')
this.parseHeaders(opts.headers);
else
throw new Error('Missing Content-Type');
}
inherits(Busboy, WritableStream);
Busboy.prototype.emit = function(ev) {
if (ev === 'finish' && !this._done)
this._parser && this._parser.end();
else
WritableStream.prototype.emit.apply(this, arguments);
};
Busboy.prototype.parseHeaders = function(headers) {
this._parser = undefined;
if (headers['content-type']) {
var parsed = parseParams(headers['content-type']),
matched, type;
for (var i = 0; i < TYPES_LEN; ++i) {
type = TYPES[i];
if (typeof type.detect === 'function')
matched = type.detect(parsed);
else
matched = type.detect.test(parsed[0]);
if (matched)
break;
}
if (matched) {
var cfg = {
limits: this.opts.limits,
headers: headers,
parsedConType: parsed,
highWaterMark: undefined,
fileHwm: undefined,
defCharset: undefined,
preservePath: false
};
if (this.opts.highWaterMark)
cfg.highWaterMark = this.opts.highWaterMark;
if (this.opts.fileHwm)
cfg.fileHwm = this.opts.fileHwm;
cfg.defCharset = this.opts.defCharset;
cfg.preservePath = this.opts.preservePath;
this._parser = type(this, cfg);
return;
}
}
throw new Error('Unsupported content type: ' + headers['content-type']);
};
Busboy.prototype._write = function(chunk, encoding, cb) {
if (!this._parser)
return cb(new Error('Not ready to parse. Missing Content-Type?'));
this._parser.write(chunk, cb);
};
var TYPES = [], TYPES_LEN = 0;
fs.readdirSync(__dirname + '/types').forEach(function(type) {
var typemod = require(__dirname + '/types/' + type);
if (typemod.detect) {
TYPES.push(typemod);
++TYPES_LEN;
}
});
module.exports = Busboy;

View File

@@ -0,0 +1,320 @@
// TODO:
// * support 1 nested multipart level
// (see second multipart example here:
// http://www.w3.org/TR/html401/interact/forms.html#didx-multipartform-data)
// * support limits.fieldNameSize
// -- this will require modifications to utils.parseParams
var ReadableStream = require('stream').Readable || require('readable-stream'),
inherits = require('util').inherits;
var Dicer = require('dicer');
var parseParams = require('../utils').parseParams,
decodeText = require('../utils').decodeText,
basename = require('../utils').basename;
var RE_BOUNDARY = /^boundary$/i,
RE_FIELD = /^form-data$/i,
RE_CHARSET = /^charset$/i,
RE_FILENAME = /^filename$/i,
RE_NAME = /^name$/i;
Multipart.detect = /^multipart\/form-data/i;
function Multipart(boy, cfg) {
if (!(this instanceof Multipart))
return new Multipart(boy, cfg);
var i,
len,
self = this,
boundary,
limits = cfg.limits,
headers = cfg.headers,
parsedConType = cfg.parsedConType || [],
defCharset = cfg.defCharset || 'utf8',
preservePath = cfg.preservePath,
fileopts = (typeof cfg.fileHwm === 'number'
? { highWaterMark: cfg.fileHwm }
: {});
for (i = 0, len = parsedConType.length; i < len; ++i) {
if (Array.isArray(parsedConType[i])
&& RE_BOUNDARY.test(parsedConType[i][0])) {
boundary = parsedConType[i][1];
break;
}
}
function checkFinished() {
if (nends === 0 && finished && !boy._done) {
finished = false;
process.nextTick(function() {
boy._done = true;
boy.emit('finish');
});
}
}
if (typeof boundary !== 'string')
throw new Error('Multipart: Boundary not found');
var fieldSizeLimit = (limits && typeof limits.fieldSize === 'number'
? limits.fieldSize
: 1 * 1024 * 1024),
fileSizeLimit = (limits && typeof limits.fileSize === 'number'
? limits.fileSize
: Infinity),
filesLimit = (limits && typeof limits.files === 'number'
? limits.files
: Infinity),
fieldsLimit = (limits && typeof limits.fields === 'number'
? limits.fields
: Infinity),
partsLimit = (limits && typeof limits.parts === 'number'
? limits.parts
: Infinity);
var nfiles = 0,
nfields = 0,
nends = 0,
curFile,
curField,
finished = false;
this._needDrain = false;
this._pause = false;
this._cb = undefined;
this._nparts = 0;
this._boy = boy;
var parserCfg = {
boundary: boundary,
maxHeaderPairs: (limits && limits.headerPairs)
};
if (fileopts.highWaterMark)
parserCfg.partHwm = fileopts.highWaterMark;
if (cfg.highWaterMark)
parserCfg.highWaterMark = cfg.highWaterMark;
this.parser = new Dicer(parserCfg);
this.parser.on('drain', function() {
self._needDrain = false;
if (self._cb && !self._pause) {
var cb = self._cb;
self._cb = undefined;
cb();
}
}).on('part', function onPart(part) {
if (++self._nparts > partsLimit) {
self.parser.removeListener('part', onPart);
self.parser.on('part', skipPart);
boy.hitPartsLimit = true;
boy.emit('partsLimit');
return skipPart(part);
}
// hack because streams2 _always_ doesn't emit 'end' until nextTick, so let
// us emit 'end' early since we know the part has ended if we are already
// seeing the next part
if (curField) {
var field = curField;
field.emit('end');
field.removeAllListeners('end');
}
part.on('header', function(header) {
var contype,
fieldname,
parsed,
charset,
encoding,
filename,
nsize = 0;
if (header['content-type']) {
parsed = parseParams(header['content-type'][0]);
if (parsed[0]) {
contype = parsed[0].toLowerCase();
for (i = 0, len = parsed.length; i < len; ++i) {
if (RE_CHARSET.test(parsed[i][0])) {
charset = parsed[i][1].toLowerCase();
break;
}
}
}
}
if (contype === undefined)
contype = 'text/plain';
if (charset === undefined)
charset = defCharset;
if (header['content-disposition']) {
parsed = parseParams(header['content-disposition'][0]);
if (!RE_FIELD.test(parsed[0]))
return skipPart(part);
for (i = 0, len = parsed.length; i < len; ++i) {
if (RE_NAME.test(parsed[i][0])) {
fieldname = decodeText(parsed[i][1], 'binary', 'utf8');
} else if (RE_FILENAME.test(parsed[i][0])) {
filename = decodeText(parsed[i][1], 'binary', 'utf8');
if (!preservePath)
filename = basename(filename);
}
}
} else
return skipPart(part);
if (header['content-transfer-encoding'])
encoding = header['content-transfer-encoding'][0].toLowerCase();
else
encoding = '7bit';
var onData,
onEnd;
if (contype === 'application/octet-stream' || filename !== undefined) {
// file/binary field
if (nfiles === filesLimit) {
if (!boy.hitFilesLimit) {
boy.hitFilesLimit = true;
boy.emit('filesLimit');
}
return skipPart(part);
}
++nfiles;
if (!boy._events.file) {
self.parser._ignore();
return;
}
++nends;
var file = new FileStream(fileopts);
curFile = file;
file.on('end', function() {
--nends;
checkFinished();
});
file._read = function(n) {
if (!self._pause)
return;
self._pause = false;
if (self._cb && !self._needDrain) {
var cb = self._cb;
self._cb = undefined;
cb();
}
};
boy.emit('file', fieldname, file, filename, encoding, contype);
onData = function(data) {
if ((nsize += data.length) > fileSizeLimit) {
var extralen = (fileSizeLimit - (nsize - data.length));
if (extralen > 0)
file.push(data.slice(0, extralen));
file.emit('limit');
file.truncated = true;
part.removeAllListeners('data');
} else if (!file.push(data))
self._pause = true;
};
onEnd = function() {
curFile = undefined;
file.push(null);
};
} else {
// non-file field
if (nfields === fieldsLimit) {
if (!boy.hitFieldsLimit) {
boy.hitFieldsLimit = true;
boy.emit('fieldsLimit');
}
return skipPart(part);
}
++nfields;
++nends;
var buffer = '',
truncated = false;
curField = part;
onData = function(data) {
if ((nsize += data.length) > fieldSizeLimit) {
var extralen = (fieldSizeLimit - (nsize - data.length));
buffer += data.toString('binary', 0, extralen);
truncated = true;
part.removeAllListeners('data');
} else
buffer += data.toString('binary');
};
onEnd = function() {
curField = undefined;
if (buffer.length)
buffer = decodeText(buffer, 'binary', charset);
boy.emit('field', fieldname, buffer, false, truncated);
--nends;
checkFinished();
};
}
/* As of node@2efe4ab761666 (v0.10.29+/v0.11.14+), busboy had become
broken. Streams2/streams3 is a huge black box of confusion, but
somehow overriding the sync state seems to fix things again (and still
seems to work for previous node versions).
*/
part._readableState.sync = false;
part.on('data', onData);
part.on('end', onEnd);
}).on('error', function(err) {
if (curFile)
curFile.emit('error', err);
});
}).on('error', function(err) {
boy.emit('error', err);
}).on('finish', function() {
finished = true;
checkFinished();
});
}
Multipart.prototype.write = function(chunk, cb) {
var r;
if ((r = this.parser.write(chunk)) && !this._pause)
cb();
else {
this._needDrain = !r;
this._cb = cb;
}
};
Multipart.prototype.end = function() {
var self = this;
if (this._nparts === 0 && !self._boy._done) {
process.nextTick(function() {
self._boy._done = true;
self._boy.emit('finish');
});
} else if (this.parser.writable)
this.parser.end();
};
function skipPart(part) {
part.resume();
}
function FileStream(opts) {
if (!(this instanceof FileStream))
return new FileStream(opts);
ReadableStream.call(this, opts);
this.truncated = false;
}
inherits(FileStream, ReadableStream);
FileStream.prototype._read = function(n) {};
module.exports = Multipart;

View File

@@ -0,0 +1,214 @@
var Decoder = require('../utils').Decoder,
decodeText = require('../utils').decodeText;
var RE_CHARSET = /^charset$/i;
UrlEncoded.detect = /^application\/x-www-form-urlencoded/i;
function UrlEncoded(boy, cfg) {
if (!(this instanceof UrlEncoded))
return new UrlEncoded(boy, cfg);
var limits = cfg.limits,
headers = cfg.headers,
parsedConType = cfg.parsedConType;
this.boy = boy;
this.fieldSizeLimit = (limits && typeof limits.fieldSize === 'number'
? limits.fieldSize
: 1 * 1024 * 1024);
this.fieldNameSizeLimit = (limits && typeof limits.fieldNameSize === 'number'
? limits.fieldNameSize
: 100);
this.fieldsLimit = (limits && typeof limits.fields === 'number'
? limits.fields
: Infinity);
var charset;
for (var i = 0, len = parsedConType.length; i < len; ++i) {
if (Array.isArray(parsedConType[i])
&& RE_CHARSET.test(parsedConType[i][0])) {
charset = parsedConType[i][1].toLowerCase();
break;
}
}
if (charset === undefined)
charset = cfg.defCharset || 'utf8';
this.decoder = new Decoder();
this.charset = charset;
this._fields = 0;
this._state = 'key';
this._checkingBytes = true;
this._bytesKey = 0;
this._bytesVal = 0;
this._key = '';
this._val = '';
this._keyTrunc = false;
this._valTrunc = false;
this._hitlimit = false;
}
UrlEncoded.prototype.write = function(data, cb) {
if (this._fields === this.fieldsLimit) {
if (!this.boy.hitFieldsLimit) {
this.boy.hitFieldsLimit = true;
this.boy.emit('fieldsLimit');
}
return cb();
}
var idxeq, idxamp, i, p = 0, len = data.length;
while (p < len) {
if (this._state === 'key') {
idxeq = idxamp = undefined;
for (i = p; i < len; ++i) {
if (!this._checkingBytes)
++p;
if (data[i] === 0x3D/*=*/) {
idxeq = i;
break;
} else if (data[i] === 0x26/*&*/) {
idxamp = i;
break;
}
if (this._checkingBytes && this._bytesKey === this.fieldNameSizeLimit) {
this._hitLimit = true;
break;
} else if (this._checkingBytes)
++this._bytesKey;
}
if (idxeq !== undefined) {
// key with assignment
if (idxeq > p)
this._key += this.decoder.write(data.toString('binary', p, idxeq));
this._state = 'val';
this._hitLimit = false;
this._checkingBytes = true;
this._val = '';
this._bytesVal = 0;
this._valTrunc = false;
this.decoder.reset();
p = idxeq + 1;
} else if (idxamp !== undefined) {
// key with no assignment
++this._fields;
var key, keyTrunc = this._keyTrunc;
if (idxamp > p)
key = (this._key += this.decoder.write(data.toString('binary', p, idxamp)));
else
key = this._key;
this._hitLimit = false;
this._checkingBytes = true;
this._key = '';
this._bytesKey = 0;
this._keyTrunc = false;
this.decoder.reset();
if (key.length) {
this.boy.emit('field', decodeText(key, 'binary', this.charset),
'',
keyTrunc,
false);
}
p = idxamp + 1;
if (this._fields === this.fieldsLimit)
return cb();
} else if (this._hitLimit) {
// we may not have hit the actual limit if there are encoded bytes...
if (i > p)
this._key += this.decoder.write(data.toString('binary', p, i));
p = i;
if ((this._bytesKey = this._key.length) === this.fieldNameSizeLimit) {
// yep, we actually did hit the limit
this._checkingBytes = false;
this._keyTrunc = true;
}
} else {
if (p < len)
this._key += this.decoder.write(data.toString('binary', p));
p = len;
}
} else {
idxamp = undefined;
for (i = p; i < len; ++i) {
if (!this._checkingBytes)
++p;
if (data[i] === 0x26/*&*/) {
idxamp = i;
break;
}
if (this._checkingBytes && this._bytesVal === this.fieldSizeLimit) {
this._hitLimit = true;
break;
}
else if (this._checkingBytes)
++this._bytesVal;
}
if (idxamp !== undefined) {
++this._fields;
if (idxamp > p)
this._val += this.decoder.write(data.toString('binary', p, idxamp));
this.boy.emit('field', decodeText(this._key, 'binary', this.charset),
decodeText(this._val, 'binary', this.charset),
this._keyTrunc,
this._valTrunc);
this._state = 'key';
this._hitLimit = false;
this._checkingBytes = true;
this._key = '';
this._bytesKey = 0;
this._keyTrunc = false;
this.decoder.reset();
p = idxamp + 1;
if (this._fields === this.fieldsLimit)
return cb();
} else if (this._hitLimit) {
// we may not have hit the actual limit if there are encoded bytes...
if (i > p)
this._val += this.decoder.write(data.toString('binary', p, i));
p = i;
if ((this._val === '' && this.fieldSizeLimit === 0)
|| (this._bytesVal = this._val.length) === this.fieldSizeLimit) {
// yep, we actually did hit the limit
this._checkingBytes = false;
this._valTrunc = true;
}
} else {
if (p < len)
this._val += this.decoder.write(data.toString('binary', p));
p = len;
}
}
}
cb();
};
UrlEncoded.prototype.end = function() {
if (this.boy._done)
return;
if (this._state === 'key' && this._key.length > 0) {
this.boy.emit('field', decodeText(this._key, 'binary', this.charset),
'',
this._keyTrunc,
false);
} else if (this._state === 'val') {
this.boy.emit('field', decodeText(this._key, 'binary', this.charset),
decodeText(this._val, 'binary', this.charset),
this._keyTrunc,
this._valTrunc);
}
this.boy._done = true;
this.boy.emit('finish');
};
module.exports = UrlEncoded;

186
node_modules/multer/node_modules/busboy/lib/utils.js generated vendored Normal file
View File

@@ -0,0 +1,186 @@
var jsencoding = require('../deps/encoding/encoding');
var RE_ENCODED = /%([a-fA-F0-9]{2})/g;
function encodedReplacer(match, byte) {
return String.fromCharCode(parseInt(byte, 16));
}
function parseParams(str) {
var res = [],
state = 'key',
charset = '',
inquote = false,
escaping = false,
p = 0,
tmp = '';
for (var i = 0, len = str.length; i < len; ++i) {
if (str[i] === '\\' && inquote) {
if (escaping)
escaping = false;
else {
escaping = true;
continue;
}
} else if (str[i] === '"') {
if (!escaping) {
if (inquote) {
inquote = false;
state = 'key';
} else
inquote = true;
continue;
} else
escaping = false;
} else {
if (escaping && inquote)
tmp += '\\';
escaping = false;
if ((state === 'charset' || state === 'lang') && str[i] === "'") {
if (state === 'charset') {
state = 'lang';
charset = tmp.substring(1);
} else
state = 'value';
tmp = '';
continue;
} else if (state === 'key'
&& (str[i] === '*' || str[i] === '=')
&& res.length) {
if (str[i] === '*')
state = 'charset';
else
state = 'value';
res[p] = [tmp, undefined];
tmp = '';
continue;
} else if (!inquote && str[i] === ';') {
state = 'key';
if (charset) {
if (tmp.length) {
tmp = decodeText(tmp.replace(RE_ENCODED, encodedReplacer),
'binary',
charset);
}
charset = '';
}
if (res[p] === undefined)
res[p] = tmp;
else
res[p][1] = tmp;
tmp = '';
++p;
continue;
} else if (!inquote && (str[i] === ' ' || str[i] === '\t'))
continue;
}
tmp += str[i];
}
if (charset && tmp.length) {
tmp = decodeText(tmp.replace(RE_ENCODED, encodedReplacer),
'binary',
charset);
}
if (res[p] === undefined) {
if (tmp)
res[p] = tmp;
} else
res[p][1] = tmp;
return res;
};
exports.parseParams = parseParams;
function decodeText(text, textEncoding, destEncoding) {
var ret;
if (text && jsencoding.encodingExists(destEncoding)) {
try {
ret = jsencoding.TextDecoder(destEncoding)
.decode(new Buffer(text, textEncoding));
} catch(e) {}
}
return (typeof ret === 'string' ? ret : text);
}
exports.decodeText = decodeText;
var HEX = [
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
], RE_PLUS = /\+/g;
function Decoder() {
this.buffer = undefined;
}
Decoder.prototype.write = function(str) {
// Replace '+' with ' ' before decoding
str = str.replace(RE_PLUS, ' ');
var res = '';
var i = 0, p = 0, len = str.length;
for (; i < len; ++i) {
if (this.buffer !== undefined) {
if (!HEX[str.charCodeAt(i)]) {
res += '%' + this.buffer;
this.buffer = undefined;
--i; // retry character
} else {
this.buffer += str[i];
++p;
if (this.buffer.length === 2) {
res += String.fromCharCode(parseInt(this.buffer, 16));
this.buffer = undefined;
}
}
} else if (str[i] === '%') {
if (i > p) {
res += str.substring(p, i);
p = i;
}
this.buffer = '';
++p;
}
}
if (p < len && this.buffer === undefined)
res += str.substring(p);
return res;
};
Decoder.prototype.reset = function() {
this.buffer = undefined;
};
exports.Decoder = Decoder;
var RE_SPLIT_POSIX =
/^(\/?|)([\s\S]*?)((?:\.{1,2}|[^\/]+?|)(\.[^.\/]*|))(?:[\/]*)$/,
RE_SPLIT_DEVICE =
/^([a-zA-Z]:|[\\\/]{2}[^\\\/]+[\\\/]+[^\\\/]+)?([\\\/])?([\s\S]*?)$/,
RE_SPLIT_WINDOWS =
/^([\s\S]*?)((?:\.{1,2}|[^\\\/]+?|)(\.[^.\/\\]*|))(?:[\\\/]*)$/;
function splitPathPosix(filename) {
return RE_SPLIT_POSIX.exec(filename).slice(1);
}
function splitPathWindows(filename) {
// Separate device+slash from tail
var result = RE_SPLIT_DEVICE.exec(filename),
device = (result[1] || '') + (result[2] || ''),
tail = result[3] || '';
// Split the tail into dir, basename and extension
var result2 = RE_SPLIT_WINDOWS.exec(tail),
dir = result2[1],
basename = result2[2],
ext = result2[3];
return [device, dir, basename, ext];
}
function basename(path) {
var f = splitPathPosix(path)[2];
if (f === path)
f = splitPathWindows(path)[2];
return f;
}
exports.basename = basename;

View File

@@ -0,0 +1,19 @@
Copyright Brian White. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

View File

@@ -0,0 +1,122 @@
Description
===========
A very fast streaming multipart parser for node.js.
Benchmarks can be found [here](https://github.com/mscdex/dicer/wiki/Benchmarks).
Requirements
============
* [node.js](http://nodejs.org/) -- v0.8.0 or newer
Install
============
npm install dicer
Examples
========
* Parse an HTTP form upload
```javascript
var inspect = require('util').inspect,
http = require('http');
var Dicer = require('dicer');
// quick and dirty way to parse multipart boundary
var RE_BOUNDARY = /^multipart\/.+?(?:; boundary=(?:(?:"(.+)")|(?:([^\s]+))))$/i,
HTML = new Buffer('<html><head></head><body>\
<form method="POST" enctype="multipart/form-data">\
<input type="text" name="textfield"><br />\
<input type="file" name="filefield"><br />\
<input type="submit">\
</form>\
</body></html>'),
PORT = 8080;
http.createServer(function(req, res) {
var m;
if (req.method === 'POST'
&& req.headers['content-type']
&& (m = RE_BOUNDARY.exec(req.headers['content-type']))) {
var d = new Dicer({ boundary: m[1] || m[2] });
d.on('part', function(p) {
console.log('New part!');
p.on('header', function(header) {
for (var h in header) {
console.log('Part header: k: ' + inspect(h)
+ ', v: ' + inspect(header[h]));
}
});
p.on('data', function(data) {
console.log('Part data: ' + inspect(data.toString()));
});
p.on('end', function() {
console.log('End of part\n');
});
});
d.on('finish', function() {
console.log('End of parts');
res.writeHead(200);
res.end('Form submission successful!');
});
req.pipe(d);
} else if (req.method === 'GET' && req.url === '/') {
res.writeHead(200);
res.end(HTML);
} else {
res.writeHead(404);
res.end();
}
}).listen(PORT, function() {
console.log('Listening for requests on port ' + PORT);
});
```
API
===
_Dicer_ is a _WritableStream_
Dicer (special) events
----------------------
* **end**() - Emitted when all parts have been parsed.
* **part**(< _PartStream_ >stream) - Emitted when a new part has been found.
* **preamble**(< _PartStream_ >stream) - Emitted for preamble if you should happen to need it (can usually be ignored).
* **trailer**(< _Buffer_ >data) - Emitted when trailing data was found after the terminating boundary (as with the preamble, this can usually be ignored too).
Dicer methods
-------------
* **(constructor)**(< _object_ >config) - Creates and returns a new Dicer instance with the following valid `config` settings:
* **boundary** - _string_ - This is the boundary used to detect the beginning of a new part.
* **headerFirst** - _boolean_ - If true, preamble header parsing will be performed first.
* **maxHeaderPairs** - _integer_ - The maximum number of header key=>value pairs to parse **Default:** 2000 (same as node's http).
* **setBoundary**(< _string_ >boundary) - _(void)_ - Sets the boundary to use for parsing and performs some initialization needed for parsing. You should only need to use this if you set `headerFirst` to true in the constructor and are parsing the boundary from the preamble header.
_PartStream_ is a _ReadableStream_
PartStream (special) events
---------------------------
* **header**(< _object_ >header) - An object containing the header for this particular part. Each property value is an _array_ of one or more string values.

View File

@@ -0,0 +1,63 @@
var assert = require('assert');
var Dicer = require('..'),
boundary = '-----------------------------168072824752491622650073',
d = new Dicer({ boundary: boundary }),
mb = 100,
buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
callbacks =
{ partBegin: -1,
partEnd: -1,
headerField: -1,
headerValue: -1,
partData: -1,
end: -1,
};
d.on('part', function(p) {
callbacks.partBegin++;
p.on('header', function(header) {
/*for (var h in header)
console.log('Part header: k: ' + inspect(h) + ', v: ' + inspect(header[h]));*/
});
p.on('data', function(data) {
callbacks.partData++;
//console.log('Part data: ' + inspect(data.toString()));
});
p.on('end', function() {
//console.log('End of part\n');
callbacks.partEnd++;
});
});
d.on('end', function() {
//console.log('End of parts');
callbacks.end++;
});
var start = +new Date(),
nparsed = d.write(buffer),
duration = +new Date - start,
mbPerSec = (mb / (duration / 1000)).toFixed(2);
console.log(mbPerSec+' mb/sec');
//assert.equal(nparsed, buffer.length);
function createMultipartBuffer(boundary, size) {
var head =
'--'+boundary+'\r\n'
+ 'content-disposition: form-data; name="field1"\r\n'
+ '\r\n'
, tail = '\r\n--'+boundary+'--\r\n'
, buffer = new Buffer(size);
buffer.write(head, 'ascii', 0);
buffer.write(tail, 'ascii', buffer.length - tail.length);
return buffer;
}
process.on('exit', function() {
/*for (var k in callbacks) {
assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
}*/
});

View File

@@ -0,0 +1,70 @@
var assert = require('assert');
require('../node_modules/formidable/test/common');
var multipartParser = require('../node_modules/formidable/lib/multipart_parser'),
MultipartParser = multipartParser.MultipartParser,
parser = new MultipartParser(),
boundary = '-----------------------------168072824752491622650073',
mb = 100,
buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
callbacks =
{ partBegin: -1,
partEnd: -1,
headerField: -1,
headerValue: -1,
partData: -1,
end: -1,
};
parser.initWithBoundary(boundary);
parser.onHeaderField = function() {
callbacks.headerField++;
};
parser.onHeaderValue = function() {
callbacks.headerValue++;
};
parser.onPartBegin = function() {
callbacks.partBegin++;
};
parser.onPartData = function() {
callbacks.partData++;
};
parser.onPartEnd = function() {
callbacks.partEnd++;
};
parser.onEnd = function() {
callbacks.end++;
};
var start = +new Date(),
nparsed = parser.write(buffer),
duration = +new Date - start,
mbPerSec = (mb / (duration / 1000)).toFixed(2);
console.log(mbPerSec+' mb/sec');
//assert.equal(nparsed, buffer.length);
function createMultipartBuffer(boundary, size) {
var head =
'--'+boundary+'\r\n'
+ 'content-disposition: form-data; name="field1"\r\n'
+ '\r\n'
, tail = '\r\n--'+boundary+'--\r\n'
, buffer = new Buffer(size);
buffer.write(head, 'ascii', 0);
buffer.write(tail, 'ascii', buffer.length - tail.length);
return buffer;
}
process.on('exit', function() {
/*for (var k in callbacks) {
assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
}*/
});

View File

@@ -0,0 +1,56 @@
var assert = require('assert');
var multipartser = require('multipartser'),
boundary = '-----------------------------168072824752491622650073',
parser = multipartser(),
mb = 100,
buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
callbacks =
{ partBegin: -1,
partEnd: -1,
headerField: -1,
headerValue: -1,
partData: -1,
end: -1,
};
parser.boundary( boundary );
parser.on( 'part', function ( part ) {
});
parser.on( 'end', function () {
//console.log( 'completed parsing' );
});
parser.on( 'error', function ( error ) {
console.error( error );
});
var start = +new Date(),
nparsed = parser.data(buffer),
nend = parser.end(),
duration = +new Date - start,
mbPerSec = (mb / (duration / 1000)).toFixed(2);
console.log(mbPerSec+' mb/sec');
//assert.equal(nparsed, buffer.length);
function createMultipartBuffer(boundary, size) {
var head =
'--'+boundary+'\r\n'
+ 'content-disposition: form-data; name="field1"\r\n'
+ '\r\n'
, tail = '\r\n--'+boundary+'--\r\n'
, buffer = new Buffer(size);
buffer.write(head, 'ascii', 0);
buffer.write(tail, 'ascii', buffer.length - tail.length);
return buffer;
}
process.on('exit', function() {
/*for (var k in callbacks) {
assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
}*/
});

View File

@@ -0,0 +1,76 @@
var assert = require('assert'),
Form = require('multiparty').Form,
boundary = '-----------------------------168072824752491622650073',
mb = 100,
buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
callbacks =
{ partBegin: -1,
partEnd: -1,
headerField: -1,
headerValue: -1,
partData: -1,
end: -1,
};
var form = new Form({ boundary: boundary });
hijack('onParseHeaderField', function() {
callbacks.headerField++;
});
hijack('onParseHeaderValue', function() {
callbacks.headerValue++;
});
hijack('onParsePartBegin', function() {
callbacks.partBegin++;
});
hijack('onParsePartData', function() {
callbacks.partData++;
});
hijack('onParsePartEnd', function() {
callbacks.partEnd++;
});
form.on('finish', function() {
callbacks.end++;
});
var start = new Date();
form.write(buffer, function(err) {
var duration = new Date() - start;
assert.ifError(err);
var mbPerSec = (mb / (duration / 1000)).toFixed(2);
console.log(mbPerSec+' mb/sec');
});
//assert.equal(nparsed, buffer.length);
function createMultipartBuffer(boundary, size) {
var head =
'--'+boundary+'\r\n'
+ 'content-disposition: form-data; name="field1"\r\n'
+ '\r\n'
, tail = '\r\n--'+boundary+'--\r\n'
, buffer = new Buffer(size);
buffer.write(head, 'ascii', 0);
buffer.write(tail, 'ascii', buffer.length - tail.length);
return buffer;
}
process.on('exit', function() {
/*for (var k in callbacks) {
assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
}*/
});
function hijack(name, fn) {
var oldFn = form[name];
form[name] = function() {
fn();
return oldFn.apply(this, arguments);
};
}

View File

@@ -0,0 +1,63 @@
// A special, edited version of the multipart parser from parted is needed here
// because otherwise it attempts to do some things above and beyond just parsing
// -- like saving to disk and whatnot
var assert = require('assert');
var Parser = require('./parted-multipart'),
boundary = '-----------------------------168072824752491622650073',
parser = new Parser('boundary=' + boundary),
mb = 100,
buffer = createMultipartBuffer(boundary, mb * 1024 * 1024),
callbacks =
{ partBegin: -1,
partEnd: -1,
headerField: -1,
headerValue: -1,
partData: -1,
end: -1,
};
parser.on('header', function() {
//callbacks.headerField++;
});
parser.on('data', function() {
//callbacks.partBegin++;
});
parser.on('part', function() {
});
parser.on('end', function() {
//callbacks.end++;
});
var start = +new Date(),
nparsed = parser.write(buffer),
duration = +new Date - start,
mbPerSec = (mb / (duration / 1000)).toFixed(2);
console.log(mbPerSec+' mb/sec');
//assert.equal(nparsed, buffer.length);
function createMultipartBuffer(boundary, size) {
var head =
'--'+boundary+'\r\n'
+ 'content-disposition: form-data; name="field1"\r\n'
+ '\r\n'
, tail = '\r\n--'+boundary+'--\r\n'
, buffer = new Buffer(size);
buffer.write(head, 'ascii', 0);
buffer.write(tail, 'ascii', buffer.length - tail.length);
return buffer;
}
process.on('exit', function() {
/*for (var k in callbacks) {
assert.equal(0, callbacks[k], k+' count off by '+callbacks[k]);
}*/
});

View File

@@ -0,0 +1,485 @@
/**
* Parted (https://github.com/chjj/parted)
* A streaming multipart state parser.
* Copyright (c) 2011, Christopher Jeffrey. (MIT Licensed)
*/
var fs = require('fs')
, path = require('path')
, EventEmitter = require('events').EventEmitter
, StringDecoder = require('string_decoder').StringDecoder
, set = require('qs').set
, each = Array.prototype.forEach;
/**
* Character Constants
*/
var DASH = '-'.charCodeAt(0)
, CR = '\r'.charCodeAt(0)
, LF = '\n'.charCodeAt(0)
, COLON = ':'.charCodeAt(0)
, SPACE = ' '.charCodeAt(0);
/**
* Parser
*/
var Parser = function(type, options) {
if (!(this instanceof Parser)) {
return new Parser(type, options);
}
EventEmitter.call(this);
this.writable = true;
this.readable = true;
this.options = options || {};
var key = grab(type, 'boundary');
if (!key) {
return this._error('No boundary key found.');
}
this.key = new Buffer('\r\n--' + key);
this._key = {};
each.call(this.key, function(ch) {
this._key[ch] = true;
}, this);
this.state = 'start';
this.pending = 0;
this.written = 0;
this.writtenDisk = 0;
this.buff = new Buffer(200);
this.preamble = true;
this.epilogue = false;
this._reset();
};
Parser.prototype.__proto__ = EventEmitter.prototype;
/**
* Parsing
*/
Parser.prototype.write = function(data) {
if (!this.writable
|| this.epilogue) return;
try {
this._parse(data);
} catch (e) {
this._error(e);
}
return true;
};
Parser.prototype.end = function(data) {
if (!this.writable) return;
if (data) this.write(data);
if (!this.epilogue) {
return this._error('Message underflow.');
}
return true;
};
Parser.prototype._parse = function(data) {
var i = 0
, len = data.length
, buff = this.buff
, key = this.key
, ch
, val
, j;
for (; i < len; i++) {
if (this.pos >= 200) {
return this._error('Potential buffer overflow.');
}
ch = data[i];
switch (this.state) {
case 'start':
switch (ch) {
case DASH:
this.pos = 3;
this.state = 'key';
break;
default:
break;
}
break;
case 'key':
if (this.pos === key.length) {
this.state = 'key_end';
i--;
} else if (ch !== key[this.pos]) {
if (this.preamble) {
this.state = 'start';
i--;
} else {
this.state = 'body';
val = this.pos - i;
if (val > 0) {
this._write(key.slice(0, val));
}
i--;
}
} else {
this.pos++;
}
break;
case 'key_end':
switch (ch) {
case CR:
this.state = 'key_line_end';
break;
case DASH:
this.state = 'key_dash_end';
break;
default:
return this._error('Expected CR or DASH.');
}
break;
case 'key_line_end':
switch (ch) {
case LF:
if (this.preamble) {
this.preamble = false;
} else {
this._finish();
}
this.state = 'header_name';
this.pos = 0;
break;
default:
return this._error('Expected CR.');
}
break;
case 'key_dash_end':
switch (ch) {
case DASH:
this.epilogue = true;
this._finish();
return;
default:
return this._error('Expected DASH.');
}
break;
case 'header_name':
switch (ch) {
case COLON:
this.header = buff.toString('ascii', 0, this.pos);
this.pos = 0;
this.state = 'header_val';
break;
default:
buff[this.pos++] = ch | 32;
break;
}
break;
case 'header_val':
switch (ch) {
case CR:
this.state = 'header_val_end';
break;
case SPACE:
if (this.pos === 0) {
break;
}
; // FALL-THROUGH
default:
buff[this.pos++] = ch;
break;
}
break;
case 'header_val_end':
switch (ch) {
case LF:
val = buff.toString('ascii', 0, this.pos);
this._header(this.header, val);
this.pos = 0;
this.state = 'header_end';
break;
default:
return this._error('Expected LF.');
}
break;
case 'header_end':
switch (ch) {
case CR:
this.state = 'head_end';
break;
default:
this.state = 'header_name';
i--;
break;
}
break;
case 'head_end':
switch (ch) {
case LF:
this.state = 'body';
i++;
if (i >= len) return;
data = data.slice(i);
i = -1;
len = data.length;
break;
default:
return this._error('Expected LF.');
}
break;
case 'body':
switch (ch) {
case CR:
if (i > 0) {
this._write(data.slice(0, i));
}
this.pos = 1;
this.state = 'key';
data = data.slice(i);
i = 0;
len = data.length;
break;
default:
// boyer-moore-like algorithm
// at felixge's suggestion
while ((j = i + key.length - 1) < len) {
if (this._key[data[j]]) break;
i = j;
}
break;
}
break;
}
}
if (this.state === 'body') {
this._write(data);
}
};
Parser.prototype._header = function(name, val) {
/*if (name === 'content-disposition') {
this.field = grab(val, 'name');
this.file = grab(val, 'filename');
if (this.file) {
this.data = stream(this.file, this.options.path);
} else {
this.decode = new StringDecoder('utf8');
this.data = '';
}
}*/
return this.emit('header', name, val);
};
Parser.prototype._write = function(data) {
/*if (this.data == null) {
return this._error('No disposition.');
}
if (this.file) {
this.data.write(data);
this.writtenDisk += data.length;
} else {
this.data += this.decode.write(data);
this.written += data.length;
}*/
this.emit('data', data);
};
Parser.prototype._reset = function() {
this.pos = 0;
this.decode = null;
this.field = null;
this.data = null;
this.file = null;
this.header = null;
};
Parser.prototype._error = function(err) {
this.destroy();
this.emit('error', typeof err === 'string'
? new Error(err)
: err);
};
Parser.prototype.destroy = function(err) {
this.writable = false;
this.readable = false;
this._reset();
};
Parser.prototype._finish = function() {
var self = this
, field = this.field
, data = this.data
, file = this.file
, part;
this.pending++;
this._reset();
if (data && data.path) {
part = data.path;
data.end(next);
} else {
part = data;
next();
}
function next() {
if (!self.readable) return;
self.pending--;
self.emit('part', field, part);
if (data && data.path) {
self.emit('file', field, part, file);
}
if (self.epilogue && !self.pending) {
self.emit('end');
self.destroy();
}
}
};
/**
* Uploads
*/
Parser.root = process.platform === 'win32'
? 'C:/Temp'
: '/tmp';
/**
* Middleware
*/
Parser.middleware = function(options) {
options = options || {};
return function(req, res, next) {
if (options.ensureBody) {
req.body = {};
}
if (req.method === 'GET'
|| req.method === 'HEAD'
|| req._multipart) return next();
req._multipart = true;
var type = req.headers['content-type'];
if (type) type = type.split(';')[0].trim().toLowerCase();
if (type === 'multipart/form-data') {
Parser.handle(req, res, next, options);
} else {
next();
}
};
};
/**
* Handler
*/
Parser.handle = function(req, res, next, options) {
var parser = new Parser(req.headers['content-type'], options)
, diskLimit = options.diskLimit
, limit = options.limit
, parts = {}
, files = {};
parser.on('error', function(err) {
req.destroy();
next(err);
});
parser.on('part', function(field, part) {
set(parts, field, part);
});
parser.on('file', function(field, path, name) {
set(files, field, {
path: path,
name: name,
toString: function() {
return path;
}
});
});
parser.on('data', function() {
if (this.writtenDisk > diskLimit || this.written > limit) {
this.emit('error', new Error('Overflow.'));
this.destroy();
}
});
parser.on('end', next);
req.body = parts;
req.files = files;
req.pipe(parser);
};
/**
* Helpers
*/
var isWindows = process.platform === 'win32';
var stream = function(name, dir) {
var ext = path.extname(name) || ''
, name = path.basename(name, ext) || ''
, dir = dir || Parser.root
, tag;
tag = Math.random().toString(36).substring(2);
name = name.substring(0, 200) + '.' + tag;
name = path.join(dir, name) + ext.substring(0, 6);
name = name.replace(/\0/g, '');
if (isWindows) {
name = name.replace(/[:*<>|"?]/g, '');
}
return fs.createWriteStream(name);
};
var grab = function(str, name) {
if (!str) return;
var rx = new RegExp('\\b' + name + '\\s*=\\s*("[^"]+"|\'[^\']+\'|[^;,]+)', 'i')
, cap = rx.exec(str);
if (cap) {
return cap[1].trim().replace(/^['"]|['"]$/g, '');
}
};
/**
* Expose
*/
module.exports = Parser;

View File

@@ -0,0 +1,225 @@
var WritableStream = require('stream').Writable
|| require('readable-stream').Writable,
inherits = require('util').inherits;
var StreamSearch = require('streamsearch');
var PartStream = require('./PartStream'),
HeaderParser = require('./HeaderParser');
var DASH = 45,
B_ONEDASH = new Buffer('-'),
B_CRLF = new Buffer('\r\n'),
EMPTY_FN = function() {};
function Dicer(cfg) {
if (!(this instanceof Dicer))
return new Dicer(cfg);
WritableStream.call(this, cfg);
if (!cfg || (!cfg.headerFirst && typeof cfg.boundary !== 'string'))
throw new TypeError('Boundary required');
if (typeof cfg.boundary === 'string')
this.setBoundary(cfg.boundary);
else
this._bparser = undefined;
this._headerFirst = cfg.headerFirst;
var self = this;
this._dashes = 0;
this._parts = 0;
this._finished = false;
this._realFinish = false;
this._isPreamble = true;
this._justMatched = false;
this._firstWrite = true;
this._inHeader = true;
this._part = undefined;
this._cb = undefined;
this._ignoreData = false;
this._partOpts = (typeof cfg.partHwm === 'number'
? { highWaterMark: cfg.partHwm }
: {});
this._pause = false;
this._hparser = new HeaderParser(cfg);
this._hparser.on('header', function(header) {
self._inHeader = false;
self._part.emit('header', header);
});
}
inherits(Dicer, WritableStream);
Dicer.prototype.emit = function(ev) {
if (ev === 'finish' && !this._realFinish) {
if (!this._finished) {
var self = this;
process.nextTick(function() {
self.emit('error', new Error('Unexpected end of multipart data'));
if (self._part && !self._ignoreData) {
var type = (self._isPreamble ? 'Preamble' : 'Part');
self._part.emit('error', new Error(type + ' terminated early due to unexpected end of multipart data'));
self._part.push(null);
process.nextTick(function() {
self._realFinish = true;
self.emit('finish');
self._realFinish = false;
});
return;
}
self._realFinish = true;
self.emit('finish');
self._realFinish = false;
});
}
} else
WritableStream.prototype.emit.apply(this, arguments);
};
Dicer.prototype._write = function(data, encoding, cb) {
// ignore unexpected data (e.g. extra trailer data after finished)
if (!this._hparser || !this._bparser)
return cb();
if (this._headerFirst && this._isPreamble) {
if (!this._part) {
this._part = new PartStream(this._partOpts);
if (this._events.preamble)
this.emit('preamble', this._part);
else
this._ignore();
}
var r = this._hparser.push(data);
if (!this._inHeader && r !== undefined && r < data.length)
data = data.slice(r);
else
return cb();
}
// allows for "easier" testing
if (this._firstWrite) {
this._bparser.push(B_CRLF);
this._firstWrite = false;
}
this._bparser.push(data);
if (this._pause)
this._cb = cb;
else
cb();
};
Dicer.prototype.reset = function() {
this._part = undefined;
this._bparser = undefined;
this._hparser = undefined;
};
Dicer.prototype.setBoundary = function(boundary) {
var self = this;
this._bparser = new StreamSearch('\r\n--' + boundary);
this._bparser.on('info', function(isMatch, data, start, end) {
self._oninfo(isMatch, data, start, end);
});
};
Dicer.prototype._ignore = function() {
if (this._part && !this._ignoreData) {
this._ignoreData = true;
this._part.on('error', EMPTY_FN);
// we must perform some kind of read on the stream even though we are
// ignoring the data, otherwise node's Readable stream will not emit 'end'
// after pushing null to the stream
this._part.resume();
}
};
Dicer.prototype._oninfo = function(isMatch, data, start, end) {
var buf, self = this, i = 0, r, ev, shouldWriteMore = true;
if (!this._part && this._justMatched && data) {
while (this._dashes < 2 && (start + i) < end) {
if (data[start + i] === DASH) {
++i;
++this._dashes;
} else {
if (this._dashes)
buf = B_ONEDASH;
this._dashes = 0;
break;
}
}
if (this._dashes === 2) {
if ((start + i) < end && this._events.trailer)
this.emit('trailer', data.slice(start + i, end));
this.reset();
this._finished = true;
}
if (this._dashes)
return;
}
if (this._justMatched)
this._justMatched = false;
if (!this._part) {
this._part = new PartStream(this._partOpts);
this._part._read = function(n) {
if (!self._pause)
return;
self._pause = false;
if (self._cb) {
var cb = self._cb;
self._cb = undefined;
cb();
}
};
ev = this._isPreamble ? 'preamble' : 'part';
if (this._events[ev])
this.emit(ev, this._part);
else
this._ignore();
if (!this._isPreamble)
this._inHeader = true;
}
if (data && start < end && !this._ignoreData) {
if (this._isPreamble || !this._inHeader) {
if (buf)
shouldWriteMore = this._part.push(buf);
shouldWriteMore = this._part.push(data.slice(start, end));
if (!shouldWriteMore)
this._pause = true;
} else if (!this._isPreamble && this._inHeader) {
if (buf)
this._hparser.push(buf);
r = this._hparser.push(data.slice(start, end));
if (!this._inHeader && r !== undefined && r < end)
this._oninfo(false, data, start + r, end);
}
}
if (isMatch) {
this._hparser.reset();
if (this._isPreamble)
this._isPreamble = false;
else {
++this._parts;
this._part.on('end', function() {
if (--self._parts === 0 && self._finished) {
self._realFinish = true;
self.emit('finish');
self._realFinish = false;
}
});
}
this._part.push(null);
this._part = undefined;
this._ignoreData = false;
this._justMatched = true;
this._dashes = 0;
}
};
module.exports = Dicer;

View File

@@ -0,0 +1,112 @@
var EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
var StreamSearch = require('streamsearch');
var B_DCRLF = new Buffer('\r\n\r\n'),
RE_CRLF = /\r\n/g,
RE_HDR = /^([^:]+):[ \t]?([\x00-\xFF]+)?$/,
MAX_HEADER_PAIRS = 2000, // from node's http.js
MAX_HEADER_SIZE = 80 * 1024; // from node's http_parser
function HeaderParser(cfg) {
if (!(this instanceof HeaderParser))
return new HeaderParser(cfg);
EventEmitter.call(this);
var self = this;
this.nread = 0;
this.maxed = false;
this.npairs = 0;
this.maxHeaderPairs = (cfg && typeof cfg.maxHeaderPairs === 'number'
? cfg.maxHeaderPairs
: MAX_HEADER_PAIRS);
this.buffer = '';
this.header = {};
this.finished = false;
this.ss = new StreamSearch(B_DCRLF);
this.ss.on('info', function(isMatch, data, start, end) {
if (data && !self.maxed) {
if (self.nread + (end - start) > MAX_HEADER_SIZE) {
end = (MAX_HEADER_SIZE - self.nread);
self.nread = MAX_HEADER_SIZE;
} else
self.nread += (end - start);
if (self.nread === MAX_HEADER_SIZE)
self.maxed = true;
self.buffer += data.toString('binary', start, end);
}
if (isMatch)
self._finish();
});
}
inherits(HeaderParser, EventEmitter);
HeaderParser.prototype.push = function(data) {
var r = this.ss.push(data);
if (this.finished)
return r;
};
HeaderParser.prototype.reset = function() {
this.finished = false;
this.buffer = '';
this.header = {};
this.ss.reset();
};
HeaderParser.prototype._finish = function() {
if (this.buffer)
this._parseHeader();
this.ss.matches = this.ss.maxMatches;
var header = this.header;
this.header = {};
this.buffer = '';
this.finished = true;
this.nread = this.npairs = 0;
this.maxed = false;
this.emit('header', header);
};
HeaderParser.prototype._parseHeader = function() {
if (this.npairs === this.maxHeaderPairs)
return;
var lines = this.buffer.split(RE_CRLF), len = lines.length, m, h,
modded = false;
for (var i = 0; i < len; ++i) {
if (lines[i].length === 0)
continue;
if (lines[i][0] === '\t' || lines[i][0] === ' ') {
// folded header content
// RFC2822 says to just remove the CRLF and not the whitespace following
// it, so we follow the RFC and include the leading whitespace ...
this.header[h][this.header[h].length - 1] += lines[i];
} else {
m = RE_HDR.exec(lines[i]);
if (m) {
h = m[1].toLowerCase();
if (m[2]) {
if (this.header[h] === undefined)
this.header[h] = [m[2]];
else
this.header[h].push(m[2]);
} else
this.header[h] = [''];
if (++this.npairs === this.maxHeaderPairs)
break;
} else {
this.buffer = lines[i];
modded = true;
break;
}
}
}
if (!modded)
this.buffer = '';
};
module.exports = HeaderParser;

View File

@@ -0,0 +1,13 @@
var inherits = require('util').inherits,
ReadableStream = require('stream').Readable || require('readable-stream');
function PartStream(opts) {
if (!(this instanceof PartStream))
return new PartStream(opts);
ReadableStream.call(this, opts);
}
inherits(PartStream, ReadableStream);
PartStream.prototype._read = function(n) {};
module.exports = PartStream;

View File

@@ -0,0 +1,19 @@
Copyright Brian White. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

View File

@@ -0,0 +1,87 @@
Description
===========
streamsearch is a module for [node.js](http://nodejs.org/) that allows searching a stream using the Boyer-Moore-Horspool algorithm.
This module is based heavily on the Streaming Boyer-Moore-Horspool C++ implementation by Hongli Lai [here](https://github.com/FooBarWidget/boyer-moore-horspool).
Requirements
============
* [node.js](http://nodejs.org/) -- v0.8.0 or newer
Installation
============
npm install streamsearch
Example
=======
```javascript
var StreamSearch = require('streamsearch'),
inspect = require('util').inspect;
var needle = new Buffer([13, 10]), // CRLF
s = new StreamSearch(needle),
chunks = [
new Buffer('foo'),
new Buffer(' bar'),
new Buffer('\r'),
new Buffer('\n'),
new Buffer('baz, hello\r'),
new Buffer('\n world.'),
new Buffer('\r\n Node.JS rules!!\r\n\r\n')
];
s.on('info', function(isMatch, data, start, end) {
if (data)
console.log('data: ' + inspect(data.toString('ascii', start, end)));
if (isMatch)
console.log('match!');
});
for (var i = 0, len = chunks.length; i < len; ++i)
s.push(chunks[i]);
// output:
//
// data: 'foo'
// data: ' bar'
// match!
// data: 'baz, hello'
// match!
// data: ' world.'
// match!
// data: ' Node.JS rules!!'
// match!
// data: ''
// match!
```
API
===
Events
------
* **info**(< _boolean_ >isMatch[, < _Buffer_ >chunk, < _integer_ >start, < _integer_ >end]) - A match _may_ or _may not_ have been made. In either case, a preceding `chunk` of data _may_ be available that did not match the needle. Data (if available) is in `chunk` between `start` (inclusive) and `end` (exclusive).
Properties
----------
* **maxMatches** - < _integer_ > - The maximum number of matches. Defaults to Infinity.
* **matches** - < _integer_ > - The current match count.
Functions
---------
* **(constructor)**(< _mixed_ >needle) - Creates and returns a new instance for searching for a _Buffer_ or _string_ `needle`.
* **push**(< _Buffer_ >chunk) - _integer_ - Processes `chunk`. The return value is the last processed index in `chunk` + 1.
* **reset**() - _(void)_ - Resets internal state. Useful for when you wish to start searching a new/different stream for example.

View File

@@ -0,0 +1,213 @@
/*
Based heavily on the Streaming Boyer-Moore-Horspool C++ implementation
by Hongli Lai at: https://github.com/FooBarWidget/boyer-moore-horspool
*/
var EventEmitter = require('events').EventEmitter,
inherits = require('util').inherits;
function jsmemcmp(buf1, pos1, buf2, pos2, num) {
for (var i = 0; i < num; ++i, ++pos1, ++pos2)
if (buf1[pos1] !== buf2[pos2])
return false;
return true;
}
function SBMH(needle) {
if (typeof needle === 'string')
needle = new Buffer(needle);
var i, j, needle_len = needle.length;
this.maxMatches = Infinity;
this.matches = 0;
this._occ = new Array(256);
this._lookbehind_size = 0;
this._needle = needle;
this._bufpos = 0;
this._lookbehind = new Buffer(needle_len);
// Initialize occurrence table.
for (j = 0; j < 256; ++j)
this._occ[j] = needle_len;
// Populate occurrence table with analysis of the needle,
// ignoring last letter.
if (needle_len >= 1) {
for (i = 0; i < needle_len - 1; ++i)
this._occ[needle[i]] = needle_len - 1 - i;
}
}
inherits(SBMH, EventEmitter);
SBMH.prototype.reset = function() {
this._lookbehind_size = 0;
this.matches = 0;
this._bufpos = 0;
};
SBMH.prototype.push = function(chunk, pos) {
var r, chlen;
if (!Buffer.isBuffer(chunk))
chunk = new Buffer(chunk, 'binary');
chlen = chunk.length;
this._bufpos = pos || 0;
while (r !== chlen && this.matches < this.maxMatches)
r = this._sbmh_feed(chunk);
return r;
};
SBMH.prototype._sbmh_feed = function(data) {
var len = data.length, needle = this._needle, needle_len = needle.length;
// Positive: points to a position in `data`
// pos == 3 points to data[3]
// Negative: points to a position in the lookbehind buffer
// pos == -2 points to lookbehind[lookbehind_size - 2]
var pos = -this._lookbehind_size,
last_needle_char = needle[needle_len - 1],
occ = this._occ,
lookbehind = this._lookbehind;
if (pos < 0) {
// Lookbehind buffer is not empty. Perform Boyer-Moore-Horspool
// search with character lookup code that considers both the
// lookbehind buffer and the current round's haystack data.
//
// Loop until
// there is a match.
// or until
// we've moved past the position that requires the
// lookbehind buffer. In this case we switch to the
// optimized loop.
// or until
// the character to look at lies outside the haystack.
while (pos < 0 && pos <= len - needle_len) {
var ch = this._sbmh_lookup_char(data, pos + needle_len - 1);
if (ch === last_needle_char
&& this._sbmh_memcmp(data, pos, needle_len - 1)) {
this._lookbehind_size = 0;
++this.matches;
if (pos > -this._lookbehind_size)
this.emit('info', true, lookbehind, 0, this._lookbehind_size + pos);
else
this.emit('info', true);
this._bufpos = pos + needle_len;
return pos + needle_len;
} else
pos += occ[ch];
}
// No match.
if (pos < 0) {
// There's too few data for Boyer-Moore-Horspool to run,
// so let's use a different algorithm to skip as much as
// we can.
// Forward pos until
// the trailing part of lookbehind + data
// looks like the beginning of the needle
// or until
// pos == 0
while (pos < 0 && !this._sbmh_memcmp(data, pos, len - pos))
pos++;
}
if (pos >= 0) {
// Discard lookbehind buffer.
this.emit('info', false, lookbehind, 0, this._lookbehind_size);
this._lookbehind_size = 0;
} else {
// Cut off part of the lookbehind buffer that has
// been processed and append the entire haystack
// into it.
var bytesToCutOff = this._lookbehind_size + pos;
if (bytesToCutOff > 0) {
// The cut off data is guaranteed not to contain the needle.
this.emit('info', false, lookbehind, 0, bytesToCutOff);
}
lookbehind.copy(lookbehind, 0, bytesToCutOff,
this._lookbehind_size - bytesToCutOff);
this._lookbehind_size -= bytesToCutOff;
data.copy(lookbehind, this._lookbehind_size);
this._lookbehind_size += len;
this._bufpos = len;
return len;
}
}
if (pos >= 0)
pos += this._bufpos;
// Lookbehind buffer is now empty. Perform Boyer-Moore-Horspool
// search with optimized character lookup code that only considers
// the current round's haystack data.
while (pos <= len - needle_len) {
var ch = data[pos + needle_len - 1];
if (ch === last_needle_char
&& data[pos] === needle[0]
&& jsmemcmp(needle, 0, data, pos, needle_len - 1)) {
++this.matches;
if (pos > 0)
this.emit('info', true, data, this._bufpos, pos);
else
this.emit('info', true);
this._bufpos = pos + needle_len;
return pos + needle_len;
} else
pos += occ[ch];
}
// There was no match. If there's trailing haystack data that we cannot
// match yet using the Boyer-Moore-Horspool algorithm (because the trailing
// data is less than the needle size) then match using a modified
// algorithm that starts matching from the beginning instead of the end.
// Whatever trailing data is left after running this algorithm is added to
// the lookbehind buffer.
if (pos < len) {
while (pos < len && (data[pos] !== needle[0]
|| !jsmemcmp(data, pos, needle, 0, len - pos))) {
++pos;
}
if (pos < len) {
data.copy(lookbehind, 0, pos, pos + (len - pos));
this._lookbehind_size = len - pos;
}
}
// Everything until pos is guaranteed not to contain needle data.
if (pos > 0)
this.emit('info', false, data, this._bufpos, pos < len ? pos : len);
this._bufpos = len;
return len;
};
SBMH.prototype._sbmh_lookup_char = function(data, pos) {
if (pos < 0)
return this._lookbehind[this._lookbehind_size + pos];
else
return data[pos];
}
SBMH.prototype._sbmh_memcmp = function(data, pos, len) {
var i = 0;
while (i < len) {
if (this._sbmh_lookup_char(data, pos + i) === this._needle[i])
++i;
else
return false;
}
return true;
}
module.exports = SBMH;

View File

@@ -0,0 +1,57 @@
{
"name": "streamsearch",
"version": "0.1.2",
"author": {
"name": "Brian White",
"email": "mscdex@mscdex.net"
},
"description": "Streaming Boyer-Moore-Horspool searching for node.js",
"main": "./lib/sbmh",
"engines": {
"node": ">=0.8.0"
},
"keywords": [
"stream",
"horspool",
"boyer-moore-horspool",
"boyer-moore",
"search"
],
"licenses": [
{
"type": "MIT",
"url": "http://github.com/mscdex/streamsearch/raw/master/LICENSE"
}
],
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/mscdex/streamsearch.git"
},
"readme": "Description\n===========\n\nstreamsearch is a module for [node.js](http://nodejs.org/) that allows searching a stream using the Boyer-Moore-Horspool algorithm.\n\nThis module is based heavily on the Streaming Boyer-Moore-Horspool C++ implementation by Hongli Lai [here](https://github.com/FooBarWidget/boyer-moore-horspool).\n\n\nRequirements\n============\n\n* [node.js](http://nodejs.org/) -- v0.8.0 or newer\n\n\nInstallation\n============\n\n npm install streamsearch\n\nExample\n=======\n\n```javascript\n var StreamSearch = require('streamsearch'),\n inspect = require('util').inspect;\n\n var needle = new Buffer([13, 10]), // CRLF\n s = new StreamSearch(needle),\n chunks = [\n new Buffer('foo'),\n new Buffer(' bar'),\n new Buffer('\\r'),\n new Buffer('\\n'),\n new Buffer('baz, hello\\r'),\n new Buffer('\\n world.'),\n new Buffer('\\r\\n Node.JS rules!!\\r\\n\\r\\n')\n ];\n s.on('info', function(isMatch, data, start, end) {\n if (data)\n console.log('data: ' + inspect(data.toString('ascii', start, end)));\n if (isMatch)\n console.log('match!');\n });\n for (var i = 0, len = chunks.length; i < len; ++i)\n s.push(chunks[i]);\n\n // output:\n //\n // data: 'foo'\n // data: ' bar'\n // match!\n // data: 'baz, hello'\n // match!\n // data: ' world.'\n // match!\n // data: ' Node.JS rules!!'\n // match!\n // data: ''\n // match!\n```\n\n\nAPI\n===\n\nEvents\n------\n\n* **info**(< _boolean_ >isMatch[, < _Buffer_ >chunk, < _integer_ >start, < _integer_ >end]) - A match _may_ or _may not_ have been made. In either case, a preceding `chunk` of data _may_ be available that did not match the needle. Data (if available) is in `chunk` between `start` (inclusive) and `end` (exclusive).\n\n\nProperties\n----------\n\n* **maxMatches** - < _integer_ > - The maximum number of matches. Defaults to Infinity.\n\n* **matches** - < _integer_ > - The current match count.\n\n\nFunctions\n---------\n\n* **(constructor)**(< _mixed_ >needle) - Creates and returns a new instance for searching for a _Buffer_ or _string_ `needle`.\n\n* **push**(< _Buffer_ >chunk) - _integer_ - Processes `chunk`. The return value is the last processed index in `chunk` + 1.\n\n* **reset**() - _(void)_ - Resets internal state. Useful for when you wish to start searching a new/different stream for example.\n",
"readmeFilename": "README.md",
"_id": "streamsearch@0.1.2",
"dist": {
"shasum": "808b9d0e56fc273d809ba57338e929919a1a9f1a",
"tarball": "http://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz"
},
"_from": "streamsearch@0.1.2",
"_resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"scripts": {},
"_npmVersion": "1.2.18",
"_npmUser": {
"name": "mscdex",
"email": "mscdex@mscdex.net"
},
"maintainers": [
{
"name": "mscdex",
"email": "mscdex@mscdex.net"
}
],
"directories": {},
"_shasum": "808b9d0e56fc273d809ba57338e929919a1a9f1a",
"bugs": {
"url": "https://github.com/mscdex/streamsearch/issues"
},
"homepage": "https://github.com/mscdex/streamsearch#readme"
}

View File

@@ -0,0 +1,63 @@
{
"name": "dicer",
"version": "0.2.3",
"author": {
"name": "Brian White",
"email": "mscdex@mscdex.net"
},
"description": "A very fast streaming multipart parser for node.js",
"main": "./lib/Dicer",
"dependencies": {
"streamsearch": "0.1.2",
"readable-stream": "1.1.x"
},
"scripts": {
"test": "node test/test.js"
},
"engines": {
"node": ">=0.8.0"
},
"keywords": [
"parser",
"parse",
"parsing",
"multipart",
"form-data",
"streaming"
],
"licenses": [
{
"type": "MIT",
"url": "http://github.com/mscdex/dicer/raw/master/LICENSE"
}
],
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/mscdex/dicer.git"
},
"bugs": {
"url": "https://github.com/mscdex/dicer/issues"
},
"homepage": "https://github.com/mscdex/dicer",
"_id": "dicer@0.2.3",
"dist": {
"shasum": "f00281189a55c2351ef80490a4fe9fb2c59c4939",
"tarball": "http://registry.npmjs.org/dicer/-/dicer-0.2.3.tgz"
},
"_from": "dicer@0.2.3",
"_resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.3.tgz",
"_npmVersion": "1.4.3",
"_npmUser": {
"name": "mscdex",
"email": "mscdex@mscdex.net"
},
"maintainers": [
{
"name": "mscdex",
"email": "mscdex@mscdex.net"
}
],
"directories": {},
"_shasum": "f00281189a55c2351ef80490a4fe9fb2c59c4939",
"readme": "ERROR: No README data found!"
}

View File

@@ -0,0 +1,31 @@
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="_method"
put
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[blog]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[public_email]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[interests]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[bio]"
hello
"quote"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="commit"
Save
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="media"; filename=""
Content-Type: application/octet-stream

View File

@@ -0,0 +1 @@
put

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"_method\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[blog]\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[public_email]\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[interests]\""]}

View File

@@ -0,0 +1,3 @@
hello
"quote"

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[bio]\""]}

View File

@@ -0,0 +1 @@
Save

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"commit\""]}

View File

@@ -0,0 +1,2 @@
{"content-disposition": ["form-data; name=\"media\"; filename=\"\""],
"content-type": ["application/octet-stream"]}

View File

@@ -0,0 +1,32 @@
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="_method"
put
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[blog]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[public_email]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[interests]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[bio]"
hello
"quote"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="media"; filename=""
Content-Type: application/octet-stream
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="commit"
Save
------WebKitFormBoundaryWLHCs9qmcJJoyjKR--

View File

@@ -0,0 +1,33 @@
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="_method"
put
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[blog]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[public_email]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[interests]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[bio]"
hello
"quote"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="media"; filename=""
Content-Type: application/octet-stream
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="commit"
Save
------WebKitFormBoundaryWLHCs9qmcJJoyjKR--

View File

@@ -0,0 +1 @@
Preamble terminated early due to unexpected end of multipart data

View File

@@ -0,0 +1,32 @@
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="_method"
put
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[blog]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[public_email]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[interests]"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="profile[bio]"
hello
"quote"
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="media"; filename=""
Content-Type: application/octet-stream
------WebKitFormBoundaryWLHCs9qmcJJoyjKR
Content-Disposition: form-data; name="commit"
Save
------WebKitFormBoundaryWLHCs9qmcJJoyjKR--

View File

@@ -0,0 +1 @@
put

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"_method\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[blog]\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[public_email]\""]}

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[interests]\""]}

View File

@@ -0,0 +1,3 @@
hello
"quote"

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"profile[bio]\""]}

View File

@@ -0,0 +1,2 @@
{"content-disposition": ["form-data; name=\"media\"; filename=\"\""],
"content-type": ["application/octet-stream"]}

View File

@@ -0,0 +1 @@
Save

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"commit\""]}

View File

@@ -0,0 +1,24 @@
User-Agent: foo bar baz
Content-Type: multipart/form-data; boundary=AaB03x
--AaB03x
Content-Disposition: form-data; name="foo"
bar
--AaB03x
Content-Disposition: form-data; name="files"
Content-Type: multipart/mixed, boundary=BbC04y
--BbC04y
Content-Disposition: attachment; filename="file.txt"
Content-Type: text/plain
contents
--BbC04y
Content-Disposition: attachment; filename="flowers.jpg"
Content-Type: image/jpeg
Content-Transfer-Encoding: binary
contents
--BbC04y--
--AaB03x--

View File

@@ -0,0 +1 @@
bar

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"foo\""]}

View File

@@ -0,0 +1,12 @@
--BbC04y
Content-Disposition: attachment; filename="file.txt"
Content-Type: text/plain
contents
--BbC04y
Content-Disposition: attachment; filename="flowers.jpg"
Content-Type: image/jpeg
Content-Transfer-Encoding: binary
contents
--BbC04y--

View File

@@ -0,0 +1,2 @@
{"content-disposition": ["form-data; name=\"files\""],
"content-type": ["multipart/mixed, boundary=BbC04y"]}

View File

@@ -0,0 +1,2 @@
{"user-agent": ["foo bar baz"],
"content-type": ["multipart/form-data; boundary=AaB03x"]}

View File

@@ -0,0 +1,21 @@
--AaB03x
Content-Disposition: form-data; name="foo"
bar
--AaB03x
Content-Disposition: form-data; name="files"
Content-Type: multipart/mixed, boundary=BbC04y
--BbC04y
Content-Disposition: attachment; filename="file.txt"
Content-Type: text/plain
contents
--BbC04y
Content-Disposition: attachment; filename="flowers.jpg"
Content-Type: image/jpeg
Content-Transfer-Encoding: binary
contents
--BbC04y--
--AaB03x--

View File

@@ -0,0 +1 @@
bar

View File

@@ -0,0 +1 @@
{"content-disposition": ["form-data; name=\"foo\""]}

View File

@@ -0,0 +1,12 @@
--BbC04y
Content-Disposition: attachment; filename="file.txt"
Content-Type: text/plain
contents
--BbC04y
Content-Disposition: attachment; filename="flowers.jpg"
Content-Type: image/jpeg
Content-Transfer-Encoding: binary
contents
--BbC04y--

View File

@@ -0,0 +1,2 @@
{"content-disposition": ["form-data; name=\"files\""],
"content-type": ["multipart/mixed, boundary=BbC04y"]}

View File

@@ -0,0 +1,68 @@
var assert = require('assert'),
path = require('path');
var HeaderParser = require('../lib/HeaderParser');
var DCRLF = '\r\n\r\n',
MAXED_BUFFER = new Buffer(128 * 1024);
MAXED_BUFFER.fill(0x41); // 'A'
var group = path.basename(__filename, '.js') + '/';
[
{ source: DCRLF,
expected: {},
what: 'No header'
},
{ source: ['Content-Type:\t text/plain',
'Content-Length:0'
].join('\r\n') + DCRLF,
expected: {'content-type': [' text/plain'], 'content-length': ['0']},
what: 'Value spacing'
},
{ source: ['Content-Type:\r\n text/plain',
'Foo:\r\n bar\r\n baz',
].join('\r\n') + DCRLF,
expected: {'content-type': [' text/plain'], 'foo': [' bar baz']},
what: 'Folded values'
},
{ source: ['Content-Type:',
'Foo: ',
].join('\r\n') + DCRLF,
expected: {'content-type': [''], 'foo': ['']},
what: 'Empty values'
},
{ source: MAXED_BUFFER.toString('ascii') + DCRLF,
expected: {},
what: 'Max header size (single chunk)'
},
{ source: ['ABCDEFGHIJ', MAXED_BUFFER.toString('ascii'), DCRLF],
expected: {},
what: 'Max header size (multiple chunks #1)'
},
{ source: [MAXED_BUFFER.toString('ascii'), MAXED_BUFFER.toString('ascii'), DCRLF],
expected: {},
what: 'Max header size (multiple chunk #2)'
},
].forEach(function(v) {
var parser = new HeaderParser(),
fired = false;
parser.on('header', function(header) {
assert(!fired, makeMsg(v.what, 'Header event fired more than once'));
fired = true;
assert.deepEqual(header,
v.expected,
makeMsg(v.what, 'Parsed result mismatch'));
});
if (!Array.isArray(v.source))
v.source = [v.source];
v.source.forEach(function(s) {
parser.push(s);
});
assert(fired, makeMsg(v.what, 'Did not receive header from parser'));
});
function makeMsg(what, msg) {
return '[' + group + what + ']: ' + msg;
}

View File

@@ -0,0 +1,148 @@
var Dicer = require('..');
var assert = require('assert'),
fs = require('fs'),
path = require('path'),
inspect = require('util').inspect;
var FIXTURES_ROOT = __dirname + '/fixtures/';
var t = 0,
group = path.basename(__filename, '.js') + '/';
var tests = [
{ source: 'many',
opts: { boundary: '----WebKitFormBoundaryWLHCs9qmcJJoyjKR' },
chsize: 16,
nparts: 7,
what: 'Extra trailer data pushed after finished'
},
];
function next() {
if (t === tests.length)
return;
var v = tests[t],
fixtureBase = FIXTURES_ROOT + v.source,
fd,
n = 0,
buffer = new Buffer(v.chsize),
state = { parts: [] };
fd = fs.openSync(fixtureBase + '/original', 'r');
var dicer = new Dicer(v.opts),
error,
partErrors = 0,
finishes = 0;
dicer.on('part', function(p) {
var part = {
body: undefined,
bodylen: 0,
error: undefined,
header: undefined
};
p.on('header', function(h) {
part.header = h;
}).on('data', function(data) {
// make a copy because we are using readSync which re-uses a buffer ...
var copy = new Buffer(data.length);
data.copy(copy);
data = copy;
if (!part.body)
part.body = [ data ];
else
part.body.push(data);
part.bodylen += data.length;
}).on('error', function(err) {
part.error = err;
++partErrors;
}).on('end', function() {
if (part.body)
part.body = Buffer.concat(part.body, part.bodylen);
state.parts.push(part);
});
}).on('error', function(err) {
error = err;
}).on('finish', function() {
assert(finishes++ === 0, makeMsg(v.what, 'finish emitted multiple times'));
if (v.dicerError)
assert(error !== undefined, makeMsg(v.what, 'Expected error'));
else
assert(error === undefined, makeMsg(v.what, 'Unexpected error'));
if (v.events && v.events.indexOf('part') > -1) {
assert.equal(state.parts.length,
v.nparts,
makeMsg(v.what,
'Part count mismatch:\nActual: '
+ state.parts.length
+ '\nExpected: '
+ v.nparts));
if (!v.npartErrors)
v.npartErrors = 0;
assert.equal(partErrors,
v.npartErrors,
makeMsg(v.what,
'Part errors mismatch:\nActual: '
+ partErrors
+ '\nExpected: '
+ v.npartErrors));
for (var i = 0, header, body; i < v.nparts; ++i) {
if (fs.existsSync(fixtureBase + '/part' + (i+1))) {
body = fs.readFileSync(fixtureBase + '/part' + (i+1));
if (body.length === 0)
body = undefined;
} else
body = undefined;
assert.deepEqual(state.parts[i].body,
body,
makeMsg(v.what,
'Part #' + (i+1) + ' body mismatch'));
if (fs.existsSync(fixtureBase + '/part' + (i+1) + '.header')) {
header = fs.readFileSync(fixtureBase
+ '/part' + (i+1) + '.header', 'binary');
header = JSON.parse(header);
} else
header = undefined;
assert.deepEqual(state.parts[i].header,
header,
makeMsg(v.what,
'Part #' + (i+1)
+ ' parsed header mismatch:\nActual: '
+ inspect(state.parts[i].header)
+ '\nExpected: '
+ inspect(header)));
}
}
++t;
next();
});
while (true) {
n = fs.readSync(fd, buffer, 0, buffer.length, null);
if (n === 0) {
setTimeout(function() {
dicer.write('\r\n\r\n\r\n');
dicer.end();
}, 50);
break;
}
dicer.write(n === buffer.length ? buffer : buffer.slice(0, n));
}
fs.closeSync(fd);
}
next();
function makeMsg(what, msg) {
return '[' + group + what + ']: ' + msg;
}
process.on('exit', function() {
assert(t === tests.length,
makeMsg('_exit', 'Only ran ' + t + '/' + tests.length + ' tests'));
});

View File

@@ -0,0 +1,228 @@
var Dicer = require('..');
var assert = require('assert'),
fs = require('fs'),
path = require('path'),
inspect = require('util').inspect;
var FIXTURES_ROOT = __dirname + '/fixtures/';
var t = 0,
group = path.basename(__filename, '.js') + '/';
var tests = [
{ source: 'many',
opts: { boundary: '----WebKitFormBoundaryWLHCs9qmcJJoyjKR' },
chsize: 16,
nparts: 0,
what: 'No preamble or part listeners'
},
];
function next() {
if (t === tests.length)
return;
var v = tests[t],
fixtureBase = FIXTURES_ROOT + v.source,
fd,
n = 0,
buffer = new Buffer(v.chsize),
state = { done: false, parts: [], preamble: undefined };
fd = fs.openSync(fixtureBase + '/original', 'r');
var dicer = new Dicer(v.opts),
error,
partErrors = 0,
finishes = 0;
if (v.events && v.events.indexOf('preamble') > -1) {
dicer.on('preamble', function(p) {
var preamble = {
body: undefined,
bodylen: 0,
error: undefined,
header: undefined
};
p.on('header', function(h) {
preamble.header = h;
}).on('data', function(data) {
// make a copy because we are using readSync which re-uses a buffer ...
var copy = new Buffer(data.length);
data.copy(copy);
data = copy;
if (!preamble.body)
preamble.body = [ data ];
else
preamble.body.push(data);
preamble.bodylen += data.length;
}).on('error', function(err) {
preamble.error = err;
}).on('end', function() {
if (preamble.body)
preamble.body = Buffer.concat(preamble.body, preamble.bodylen);
if (preamble.body || preamble.header)
state.preamble = preamble;
});
});
}
if (v.events && v.events.indexOf('part') > -1) {
dicer.on('part', function(p) {
var part = {
body: undefined,
bodylen: 0,
error: undefined,
header: undefined
};
p.on('header', function(h) {
part.header = h;
}).on('data', function(data) {
// make a copy because we are using readSync which re-uses a buffer ...
var copy = new Buffer(data.length);
data.copy(copy);
data = copy;
if (!part.body)
part.body = [ data ];
else
part.body.push(data);
part.bodylen += data.length;
}).on('error', function(err) {
part.error = err;
++partErrors;
}).on('end', function() {
if (part.body)
part.body = Buffer.concat(part.body, part.bodylen);
state.parts.push(part);
});
});
}
dicer.on('error', function(err) {
error = err;
}).on('finish', function() {
assert(finishes++ === 0, makeMsg(v.what, 'finish emitted multiple times'));
if (v.dicerError)
assert(error !== undefined, makeMsg(v.what, 'Expected error'));
else
assert(error === undefined, makeMsg(v.what, 'Unexpected error'));
if (v.events && v.events.indexOf('preamble') > -1) {
var preamble;
if (fs.existsSync(fixtureBase + '/preamble')) {
var prebody = fs.readFileSync(fixtureBase + '/preamble');
if (prebody.length) {
preamble = {
body: prebody,
bodylen: prebody.length,
error: undefined,
header: undefined
};
}
}
if (fs.existsSync(fixtureBase + '/preamble.header')) {
var prehead = JSON.parse(fs.readFileSync(fixtureBase
+ '/preamble.header', 'binary'));
if (!preamble) {
preamble = {
body: undefined,
bodylen: 0,
error: undefined,
header: prehead
};
} else
preamble.header = prehead;
}
if (fs.existsSync(fixtureBase + '/preamble.error')) {
var err = new Error(fs.readFileSync(fixtureBase
+ '/preamble.error', 'binary'));
if (!preamble) {
preamble = {
body: undefined,
bodylen: 0,
error: err,
header: undefined
};
} else
preamble.error = err;
}
assert.deepEqual(state.preamble,
preamble,
makeMsg(v.what,
'Preamble mismatch:\nActual:'
+ inspect(state.preamble)
+ '\nExpected: '
+ inspect(preamble)));
}
if (v.events && v.events.indexOf('part') > -1) {
assert.equal(state.parts.length,
v.nparts,
makeMsg(v.what,
'Part count mismatch:\nActual: '
+ state.parts.length
+ '\nExpected: '
+ v.nparts));
if (!v.npartErrors)
v.npartErrors = 0;
assert.equal(partErrors,
v.npartErrors,
makeMsg(v.what,
'Part errors mismatch:\nActual: '
+ partErrors
+ '\nExpected: '
+ v.npartErrors));
for (var i = 0, header, body; i < v.nparts; ++i) {
if (fs.existsSync(fixtureBase + '/part' + (i+1))) {
body = fs.readFileSync(fixtureBase + '/part' + (i+1));
if (body.length === 0)
body = undefined;
} else
body = undefined;
assert.deepEqual(state.parts[i].body,
body,
makeMsg(v.what,
'Part #' + (i+1) + ' body mismatch'));
if (fs.existsSync(fixtureBase + '/part' + (i+1) + '.header')) {
header = fs.readFileSync(fixtureBase
+ '/part' + (i+1) + '.header', 'binary');
header = JSON.parse(header);
} else
header = undefined;
assert.deepEqual(state.parts[i].header,
header,
makeMsg(v.what,
'Part #' + (i+1)
+ ' parsed header mismatch:\nActual: '
+ inspect(state.parts[i].header)
+ '\nExpected: '
+ inspect(header)));
}
}
++t;
next();
});
while (true) {
n = fs.readSync(fd, buffer, 0, buffer.length, null);
if (n === 0) {
dicer.end();
break;
}
dicer.write(n === buffer.length ? buffer : buffer.slice(0, n));
}
fs.closeSync(fd);
}
next();
function makeMsg(what, msg) {
return '[' + group + what + ']: ' + msg;
}
process.on('exit', function() {
assert(t === tests.length,
makeMsg('_exit', 'Only ran ' + t + '/' + tests.length + ' tests'));
});

View File

@@ -0,0 +1,246 @@
var Dicer = require('..');
var assert = require('assert'),
fs = require('fs'),
path = require('path'),
inspect = require('util').inspect;
var FIXTURES_ROOT = __dirname + '/fixtures/';
var t = 0,
group = path.basename(__filename, '.js') + '/';
var tests = [
{ source: 'nested',
opts: { boundary: 'AaB03x' },
chsize: 32,
nparts: 2,
what: 'One nested multipart'
},
{ source: 'many',
opts: { boundary: '----WebKitFormBoundaryWLHCs9qmcJJoyjKR' },
chsize: 16,
nparts: 7,
what: 'Many parts'
},
{ source: 'many-wrongboundary',
opts: { boundary: 'LOLOLOL' },
chsize: 8,
nparts: 0,
dicerError: true,
what: 'Many parts, wrong boundary'
},
{ source: 'many-noend',
opts: { boundary: '----WebKitFormBoundaryWLHCs9qmcJJoyjKR' },
chsize: 16,
nparts: 7,
npartErrors: 1,
dicerError: true,
what: 'Many parts, end boundary missing, 1 file open'
},
{ source: 'nested-full',
opts: { boundary: 'AaB03x', headerFirst: true },
chsize: 32,
nparts: 2,
what: 'One nested multipart with preceding header'
},
];
function next() {
if (t === tests.length)
return;
var v = tests[t],
fixtureBase = FIXTURES_ROOT + v.source,
fd,
n = 0,
buffer = new Buffer(v.chsize),
state = { done: false, parts: [], preamble: undefined };
fd = fs.openSync(fixtureBase + '/original', 'r');
var dicer = new Dicer(v.opts),
error,
partErrors = 0,
finishes = 0;
dicer.on('preamble', function(p) {
var preamble = {
body: undefined,
bodylen: 0,
error: undefined,
header: undefined
};
p.on('header', function(h) {
preamble.header = h;
}).on('data', function(data) {
// make a copy because we are using readSync which re-uses a buffer ...
var copy = new Buffer(data.length);
data.copy(copy);
data = copy;
if (!preamble.body)
preamble.body = [ data ];
else
preamble.body.push(data);
preamble.bodylen += data.length;
}).on('error', function(err) {
preamble.error = err;
}).on('end', function() {
if (preamble.body)
preamble.body = Buffer.concat(preamble.body, preamble.bodylen);
if (preamble.body || preamble.header)
state.preamble = preamble;
});
});
dicer.on('part', function(p) {
var part = {
body: undefined,
bodylen: 0,
error: undefined,
header: undefined
};
p.on('header', function(h) {
part.header = h;
}).on('data', function(data) {
// make a copy because we are using readSync which re-uses a buffer ...
var copy = new Buffer(data.length);
data.copy(copy);
data = copy;
if (!part.body)
part.body = [ data ];
else
part.body.push(data);
part.bodylen += data.length;
}).on('error', function(err) {
part.error = err;
++partErrors;
}).on('end', function() {
if (part.body)
part.body = Buffer.concat(part.body, part.bodylen);
state.parts.push(part);
});
}).on('error', function(err) {
error = err;
}).on('finish', function() {
assert(finishes++ === 0, makeMsg(v.what, 'finish emitted multiple times'));
if (v.dicerError)
assert(error !== undefined, makeMsg(v.what, 'Expected error'));
else
assert(error === undefined, makeMsg(v.what, 'Unexpected error'));
var preamble;
if (fs.existsSync(fixtureBase + '/preamble')) {
var prebody = fs.readFileSync(fixtureBase + '/preamble');
if (prebody.length) {
preamble = {
body: prebody,
bodylen: prebody.length,
error: undefined,
header: undefined
};
}
}
if (fs.existsSync(fixtureBase + '/preamble.header')) {
var prehead = JSON.parse(fs.readFileSync(fixtureBase
+ '/preamble.header', 'binary'));
if (!preamble) {
preamble = {
body: undefined,
bodylen: 0,
error: undefined,
header: prehead
};
} else
preamble.header = prehead;
}
if (fs.existsSync(fixtureBase + '/preamble.error')) {
var err = new Error(fs.readFileSync(fixtureBase
+ '/preamble.error', 'binary'));
if (!preamble) {
preamble = {
body: undefined,
bodylen: 0,
error: err,
header: undefined
};
} else
preamble.error = err;
}
assert.deepEqual(state.preamble,
preamble,
makeMsg(v.what,
'Preamble mismatch:\nActual:'
+ inspect(state.preamble)
+ '\nExpected: '
+ inspect(preamble)));
assert.equal(state.parts.length,
v.nparts,
makeMsg(v.what,
'Part count mismatch:\nActual: '
+ state.parts.length
+ '\nExpected: '
+ v.nparts));
if (!v.npartErrors)
v.npartErrors = 0;
assert.equal(partErrors,
v.npartErrors,
makeMsg(v.what,
'Part errors mismatch:\nActual: '
+ partErrors
+ '\nExpected: '
+ v.npartErrors));
for (var i = 0, header, body; i < v.nparts; ++i) {
if (fs.existsSync(fixtureBase + '/part' + (i+1))) {
body = fs.readFileSync(fixtureBase + '/part' + (i+1));
if (body.length === 0)
body = undefined;
} else
body = undefined;
assert.deepEqual(state.parts[i].body,
body,
makeMsg(v.what,
'Part #' + (i+1) + ' body mismatch'));
if (fs.existsSync(fixtureBase + '/part' + (i+1) + '.header')) {
header = fs.readFileSync(fixtureBase
+ '/part' + (i+1) + '.header', 'binary');
header = JSON.parse(header);
} else
header = undefined;
assert.deepEqual(state.parts[i].header,
header,
makeMsg(v.what,
'Part #' + (i+1)
+ ' parsed header mismatch:\nActual: '
+ inspect(state.parts[i].header)
+ '\nExpected: '
+ inspect(header)));
}
++t;
next();
});
while (true) {
n = fs.readSync(fd, buffer, 0, buffer.length, null);
if (n === 0) {
dicer.end();
break;
}
dicer.write(n === buffer.length ? buffer : buffer.slice(0, n));
}
fs.closeSync(fd);
}
next();
function makeMsg(what, msg) {
return '[' + group + what + ']: ' + msg;
}
process.on('exit', function() {
assert(t === tests.length,
makeMsg('_exit', 'Only ran ' + t + '/' + tests.length + ' tests'));
});

View File

@@ -0,0 +1,4 @@
require('fs').readdirSync(__dirname).forEach(function(f) {
if (f.substr(0, 5) === 'test-')
require('./' + f);
});

View File

@@ -0,0 +1,5 @@
build/
test/
examples/
fs.js
zlib.js

View File

@@ -0,0 +1,18 @@
Copyright Joyent, Inc. and other Node contributors. All rights reserved.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
IN THE SOFTWARE.

View File

@@ -0,0 +1,15 @@
# readable-stream
***Node-core streams for userland***
[![NPM](https://nodei.co/npm/readable-stream.png?downloads=true&downloadRank=true)](https://nodei.co/npm/readable-stream/)
[![NPM](https://nodei.co/npm-dl/readable-stream.png&months=6&height=3)](https://nodei.co/npm/readable-stream/)
This package is a mirror of the Streams2 and Streams3 implementations in Node-core.
If you want to guarantee a stable streams base, regardless of what version of Node you, or the users of your libraries are using, use **readable-stream** *only* and avoid the *"stream"* module in Node-core.
**readable-stream** comes in two major versions, v1.0.x and v1.1.x. The former tracks the Streams2 implementation in Node 0.10, including bug-fixes and minor improvements as they are added. The latter tracks Streams3 as it develops in Node 0.11; we will likely see a v1.2.x branch for Node 0.12.
**readable-stream** uses proper patch-level versioning so if you pin to `"~1.0.0"` youll get the latest Node 0.10 Streams2 implementation, including any fixes and minor non-breaking improvements. The patch-level versions of 1.0.x and 1.1.x should mirror the patch-level versions of Node-core releases. You should prefer the **1.0.x** releases for now and when youre ready to start using Streams3, pin to `"~1.1.0"`

View File

@@ -0,0 +1 @@
module.exports = require("./lib/_stream_duplex.js")

View File

@@ -0,0 +1,923 @@
diff --git a/lib/_stream_duplex.js b/lib/_stream_duplex.js
index c5a741c..a2e0d8e 100644
--- a/lib/_stream_duplex.js
+++ b/lib/_stream_duplex.js
@@ -26,8 +26,8 @@
module.exports = Duplex;
var util = require('util');
-var Readable = require('_stream_readable');
-var Writable = require('_stream_writable');
+var Readable = require('./_stream_readable');
+var Writable = require('./_stream_writable');
util.inherits(Duplex, Readable);
diff --git a/lib/_stream_passthrough.js b/lib/_stream_passthrough.js
index a5e9864..330c247 100644
--- a/lib/_stream_passthrough.js
+++ b/lib/_stream_passthrough.js
@@ -25,7 +25,7 @@
module.exports = PassThrough;
-var Transform = require('_stream_transform');
+var Transform = require('./_stream_transform');
var util = require('util');
util.inherits(PassThrough, Transform);
diff --git a/lib/_stream_readable.js b/lib/_stream_readable.js
index 0c3fe3e..90a8298 100644
--- a/lib/_stream_readable.js
+++ b/lib/_stream_readable.js
@@ -23,10 +23,34 @@ module.exports = Readable;
Readable.ReadableState = ReadableState;
var EE = require('events').EventEmitter;
+if (!EE.listenerCount) EE.listenerCount = function(emitter, type) {
+ return emitter.listeners(type).length;
+};
+
+if (!global.setImmediate) global.setImmediate = function setImmediate(fn) {
+ return setTimeout(fn, 0);
+};
+if (!global.clearImmediate) global.clearImmediate = function clearImmediate(i) {
+ return clearTimeout(i);
+};
+
var Stream = require('stream');
var util = require('util');
+if (!util.isUndefined) {
+ var utilIs = require('core-util-is');
+ for (var f in utilIs) {
+ util[f] = utilIs[f];
+ }
+}
var StringDecoder;
-var debug = util.debuglog('stream');
+var debug;
+if (util.debuglog)
+ debug = util.debuglog('stream');
+else try {
+ debug = require('debuglog')('stream');
+} catch (er) {
+ debug = function() {};
+}
util.inherits(Readable, Stream);
@@ -380,7 +404,7 @@ function chunkInvalid(state, chunk) {
function onEofChunk(stream, state) {
- if (state.decoder && !state.ended) {
+ if (state.decoder && !state.ended && state.decoder.end) {
var chunk = state.decoder.end();
if (chunk && chunk.length) {
state.buffer.push(chunk);
diff --git a/lib/_stream_transform.js b/lib/_stream_transform.js
index b1f9fcc..b0caf57 100644
--- a/lib/_stream_transform.js
+++ b/lib/_stream_transform.js
@@ -64,8 +64,14 @@
module.exports = Transform;
-var Duplex = require('_stream_duplex');
+var Duplex = require('./_stream_duplex');
var util = require('util');
+if (!util.isUndefined) {
+ var utilIs = require('core-util-is');
+ for (var f in utilIs) {
+ util[f] = utilIs[f];
+ }
+}
util.inherits(Transform, Duplex);
diff --git a/lib/_stream_writable.js b/lib/_stream_writable.js
index ba2e920..f49288b 100644
--- a/lib/_stream_writable.js
+++ b/lib/_stream_writable.js
@@ -27,6 +27,12 @@ module.exports = Writable;
Writable.WritableState = WritableState;
var util = require('util');
+if (!util.isUndefined) {
+ var utilIs = require('core-util-is');
+ for (var f in utilIs) {
+ util[f] = utilIs[f];
+ }
+}
var Stream = require('stream');
util.inherits(Writable, Stream);
@@ -119,7 +125,7 @@ function WritableState(options, stream) {
function Writable(options) {
// Writable ctor is applied to Duplexes, though they're not
// instanceof Writable, they're instanceof Readable.
- if (!(this instanceof Writable) && !(this instanceof Stream.Duplex))
+ if (!(this instanceof Writable) && !(this instanceof require('./_stream_duplex')))
return new Writable(options);
this._writableState = new WritableState(options, this);
diff --git a/test/simple/test-stream-big-push.js b/test/simple/test-stream-big-push.js
index e3787e4..8cd2127 100644
--- a/test/simple/test-stream-big-push.js
+++ b/test/simple/test-stream-big-push.js
@@ -21,7 +21,7 @@
var common = require('../common');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
var str = 'asdfasdfasdfasdfasdf';
var r = new stream.Readable({
diff --git a/test/simple/test-stream-end-paused.js b/test/simple/test-stream-end-paused.js
index bb73777..d40efc7 100644
--- a/test/simple/test-stream-end-paused.js
+++ b/test/simple/test-stream-end-paused.js
@@ -25,7 +25,7 @@ var gotEnd = false;
// Make sure we don't miss the end event for paused 0-length streams
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var stream = new Readable();
var calledRead = false;
stream._read = function() {
diff --git a/test/simple/test-stream-pipe-after-end.js b/test/simple/test-stream-pipe-after-end.js
index b46ee90..0be8366 100644
--- a/test/simple/test-stream-pipe-after-end.js
+++ b/test/simple/test-stream-pipe-after-end.js
@@ -22,8 +22,8 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('_stream_readable');
-var Writable = require('_stream_writable');
+var Readable = require('../../lib/_stream_readable');
+var Writable = require('../../lib/_stream_writable');
var util = require('util');
util.inherits(TestReadable, Readable);
diff --git a/test/simple/test-stream-pipe-cleanup.js b/test/simple/test-stream-pipe-cleanup.js
deleted file mode 100644
index f689358..0000000
--- a/test/simple/test-stream-pipe-cleanup.js
+++ /dev/null
@@ -1,122 +0,0 @@
-// Copyright Joyent, Inc. and other Node contributors.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a
-// copy of this software and associated documentation files (the
-// "Software"), to deal in the Software without restriction, including
-// without limitation the rights to use, copy, modify, merge, publish,
-// distribute, sublicense, and/or sell copies of the Software, and to permit
-// persons to whom the Software is furnished to do so, subject to the
-// following conditions:
-//
-// The above copyright notice and this permission notice shall be included
-// in all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
-// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
-// USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-// This test asserts that Stream.prototype.pipe does not leave listeners
-// hanging on the source or dest.
-
-var common = require('../common');
-var stream = require('stream');
-var assert = require('assert');
-var util = require('util');
-
-function Writable() {
- this.writable = true;
- this.endCalls = 0;
- stream.Stream.call(this);
-}
-util.inherits(Writable, stream.Stream);
-Writable.prototype.end = function() {
- this.endCalls++;
-};
-
-Writable.prototype.destroy = function() {
- this.endCalls++;
-};
-
-function Readable() {
- this.readable = true;
- stream.Stream.call(this);
-}
-util.inherits(Readable, stream.Stream);
-
-function Duplex() {
- this.readable = true;
- Writable.call(this);
-}
-util.inherits(Duplex, Writable);
-
-var i = 0;
-var limit = 100;
-
-var w = new Writable();
-
-var r;
-
-for (i = 0; i < limit; i++) {
- r = new Readable();
- r.pipe(w);
- r.emit('end');
-}
-assert.equal(0, r.listeners('end').length);
-assert.equal(limit, w.endCalls);
-
-w.endCalls = 0;
-
-for (i = 0; i < limit; i++) {
- r = new Readable();
- r.pipe(w);
- r.emit('close');
-}
-assert.equal(0, r.listeners('close').length);
-assert.equal(limit, w.endCalls);
-
-w.endCalls = 0;
-
-r = new Readable();
-
-for (i = 0; i < limit; i++) {
- w = new Writable();
- r.pipe(w);
- w.emit('close');
-}
-assert.equal(0, w.listeners('close').length);
-
-r = new Readable();
-w = new Writable();
-var d = new Duplex();
-r.pipe(d); // pipeline A
-d.pipe(w); // pipeline B
-assert.equal(r.listeners('end').length, 2); // A.onend, A.cleanup
-assert.equal(r.listeners('close').length, 2); // A.onclose, A.cleanup
-assert.equal(d.listeners('end').length, 2); // B.onend, B.cleanup
-assert.equal(d.listeners('close').length, 3); // A.cleanup, B.onclose, B.cleanup
-assert.equal(w.listeners('end').length, 0);
-assert.equal(w.listeners('close').length, 1); // B.cleanup
-
-r.emit('end');
-assert.equal(d.endCalls, 1);
-assert.equal(w.endCalls, 0);
-assert.equal(r.listeners('end').length, 0);
-assert.equal(r.listeners('close').length, 0);
-assert.equal(d.listeners('end').length, 2); // B.onend, B.cleanup
-assert.equal(d.listeners('close').length, 2); // B.onclose, B.cleanup
-assert.equal(w.listeners('end').length, 0);
-assert.equal(w.listeners('close').length, 1); // B.cleanup
-
-d.emit('end');
-assert.equal(d.endCalls, 1);
-assert.equal(w.endCalls, 1);
-assert.equal(r.listeners('end').length, 0);
-assert.equal(r.listeners('close').length, 0);
-assert.equal(d.listeners('end').length, 0);
-assert.equal(d.listeners('close').length, 0);
-assert.equal(w.listeners('end').length, 0);
-assert.equal(w.listeners('close').length, 0);
diff --git a/test/simple/test-stream-pipe-error-handling.js b/test/simple/test-stream-pipe-error-handling.js
index c5d724b..c7d6b7d 100644
--- a/test/simple/test-stream-pipe-error-handling.js
+++ b/test/simple/test-stream-pipe-error-handling.js
@@ -21,7 +21,7 @@
var common = require('../common');
var assert = require('assert');
-var Stream = require('stream').Stream;
+var Stream = require('../../').Stream;
(function testErrorListenerCatches() {
var source = new Stream();
diff --git a/test/simple/test-stream-pipe-event.js b/test/simple/test-stream-pipe-event.js
index cb9d5fe..56f8d61 100644
--- a/test/simple/test-stream-pipe-event.js
+++ b/test/simple/test-stream-pipe-event.js
@@ -20,7 +20,7 @@
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var common = require('../common');
-var stream = require('stream');
+var stream = require('../../');
var assert = require('assert');
var util = require('util');
diff --git a/test/simple/test-stream-push-order.js b/test/simple/test-stream-push-order.js
index f2e6ec2..a5c9bf9 100644
--- a/test/simple/test-stream-push-order.js
+++ b/test/simple/test-stream-push-order.js
@@ -20,7 +20,7 @@
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var common = require('../common.js');
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var assert = require('assert');
var s = new Readable({
diff --git a/test/simple/test-stream-push-strings.js b/test/simple/test-stream-push-strings.js
index 06f43dc..1701a9a 100644
--- a/test/simple/test-stream-push-strings.js
+++ b/test/simple/test-stream-push-strings.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var util = require('util');
util.inherits(MyStream, Readable);
diff --git a/test/simple/test-stream-readable-event.js b/test/simple/test-stream-readable-event.js
index ba6a577..a8e6f7b 100644
--- a/test/simple/test-stream-readable-event.js
+++ b/test/simple/test-stream-readable-event.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
(function first() {
// First test, not reading when the readable is added.
diff --git a/test/simple/test-stream-readable-flow-recursion.js b/test/simple/test-stream-readable-flow-recursion.js
index 2891ad6..11689ba 100644
--- a/test/simple/test-stream-readable-flow-recursion.js
+++ b/test/simple/test-stream-readable-flow-recursion.js
@@ -27,7 +27,7 @@ var assert = require('assert');
// more data continuously, but without triggering a nextTick
// warning or RangeError.
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
// throw an error if we trigger a nextTick warning.
process.throwDeprecation = true;
diff --git a/test/simple/test-stream-unshift-empty-chunk.js b/test/simple/test-stream-unshift-empty-chunk.js
index 0c96476..7827538 100644
--- a/test/simple/test-stream-unshift-empty-chunk.js
+++ b/test/simple/test-stream-unshift-empty-chunk.js
@@ -24,7 +24,7 @@ var assert = require('assert');
// This test verifies that stream.unshift(Buffer(0)) or
// stream.unshift('') does not set state.reading=false.
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var r = new Readable();
var nChunks = 10;
diff --git a/test/simple/test-stream-unshift-read-race.js b/test/simple/test-stream-unshift-read-race.js
index 83fd9fa..17c18aa 100644
--- a/test/simple/test-stream-unshift-read-race.js
+++ b/test/simple/test-stream-unshift-read-race.js
@@ -29,7 +29,7 @@ var assert = require('assert');
// 3. push() after the EOF signaling null is an error.
// 4. _read() is not called after pushing the EOF null chunk.
-var stream = require('stream');
+var stream = require('../../');
var hwm = 10;
var r = stream.Readable({ highWaterMark: hwm });
var chunks = 10;
@@ -51,7 +51,14 @@ r._read = function(n) {
function push(fast) {
assert(!pushedNull, 'push() after null push');
- var c = pos >= data.length ? null : data.slice(pos, pos + n);
+ var c;
+ if (pos >= data.length)
+ c = null;
+ else {
+ if (n + pos > data.length)
+ n = data.length - pos;
+ c = data.slice(pos, pos + n);
+ }
pushedNull = c === null;
if (fast) {
pos += n;
diff --git a/test/simple/test-stream-writev.js b/test/simple/test-stream-writev.js
index 5b49e6e..b5321f3 100644
--- a/test/simple/test-stream-writev.js
+++ b/test/simple/test-stream-writev.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
var queue = [];
for (var decode = 0; decode < 2; decode++) {
diff --git a/test/simple/test-stream2-basic.js b/test/simple/test-stream2-basic.js
index 3814bf0..248c1be 100644
--- a/test/simple/test-stream2-basic.js
+++ b/test/simple/test-stream2-basic.js
@@ -21,7 +21,7 @@
var common = require('../common.js');
-var R = require('_stream_readable');
+var R = require('../../lib/_stream_readable');
var assert = require('assert');
var util = require('util');
diff --git a/test/simple/test-stream2-compatibility.js b/test/simple/test-stream2-compatibility.js
index 6cdd4e9..f0fa84b 100644
--- a/test/simple/test-stream2-compatibility.js
+++ b/test/simple/test-stream2-compatibility.js
@@ -21,7 +21,7 @@
var common = require('../common.js');
-var R = require('_stream_readable');
+var R = require('../../lib/_stream_readable');
var assert = require('assert');
var util = require('util');
diff --git a/test/simple/test-stream2-finish-pipe.js b/test/simple/test-stream2-finish-pipe.js
index 39b274f..006a19b 100644
--- a/test/simple/test-stream2-finish-pipe.js
+++ b/test/simple/test-stream2-finish-pipe.js
@@ -20,7 +20,7 @@
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var common = require('../common.js');
-var stream = require('stream');
+var stream = require('../../');
var Buffer = require('buffer').Buffer;
var r = new stream.Readable();
diff --git a/test/simple/test-stream2-fs.js b/test/simple/test-stream2-fs.js
deleted file mode 100644
index e162406..0000000
--- a/test/simple/test-stream2-fs.js
+++ /dev/null
@@ -1,72 +0,0 @@
-// Copyright Joyent, Inc. and other Node contributors.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a
-// copy of this software and associated documentation files (the
-// "Software"), to deal in the Software without restriction, including
-// without limitation the rights to use, copy, modify, merge, publish,
-// distribute, sublicense, and/or sell copies of the Software, and to permit
-// persons to whom the Software is furnished to do so, subject to the
-// following conditions:
-//
-// The above copyright notice and this permission notice shall be included
-// in all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
-// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
-// USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
-var common = require('../common.js');
-var R = require('_stream_readable');
-var assert = require('assert');
-
-var fs = require('fs');
-var FSReadable = fs.ReadStream;
-
-var path = require('path');
-var file = path.resolve(common.fixturesDir, 'x1024.txt');
-
-var size = fs.statSync(file).size;
-
-var expectLengths = [1024];
-
-var util = require('util');
-var Stream = require('stream');
-
-util.inherits(TestWriter, Stream);
-
-function TestWriter() {
- Stream.apply(this);
- this.buffer = [];
- this.length = 0;
-}
-
-TestWriter.prototype.write = function(c) {
- this.buffer.push(c.toString());
- this.length += c.length;
- return true;
-};
-
-TestWriter.prototype.end = function(c) {
- if (c) this.buffer.push(c.toString());
- this.emit('results', this.buffer);
-}
-
-var r = new FSReadable(file);
-var w = new TestWriter();
-
-w.on('results', function(res) {
- console.error(res, w.length);
- assert.equal(w.length, size);
- var l = 0;
- assert.deepEqual(res.map(function (c) {
- return c.length;
- }), expectLengths);
- console.log('ok');
-});
-
-r.pipe(w);
diff --git a/test/simple/test-stream2-httpclient-response-end.js b/test/simple/test-stream2-httpclient-response-end.js
deleted file mode 100644
index 15cffc2..0000000
--- a/test/simple/test-stream2-httpclient-response-end.js
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright Joyent, Inc. and other Node contributors.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a
-// copy of this software and associated documentation files (the
-// "Software"), to deal in the Software without restriction, including
-// without limitation the rights to use, copy, modify, merge, publish,
-// distribute, sublicense, and/or sell copies of the Software, and to permit
-// persons to whom the Software is furnished to do so, subject to the
-// following conditions:
-//
-// The above copyright notice and this permission notice shall be included
-// in all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
-// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
-// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
-// USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-var common = require('../common.js');
-var assert = require('assert');
-var http = require('http');
-var msg = 'Hello';
-var readable_event = false;
-var end_event = false;
-var server = http.createServer(function(req, res) {
- res.writeHead(200, {'Content-Type': 'text/plain'});
- res.end(msg);
-}).listen(common.PORT, function() {
- http.get({port: common.PORT}, function(res) {
- var data = '';
- res.on('readable', function() {
- console.log('readable event');
- readable_event = true;
- data += res.read();
- });
- res.on('end', function() {
- console.log('end event');
- end_event = true;
- assert.strictEqual(msg, data);
- server.close();
- });
- });
-});
-
-process.on('exit', function() {
- assert(readable_event);
- assert(end_event);
-});
-
diff --git a/test/simple/test-stream2-large-read-stall.js b/test/simple/test-stream2-large-read-stall.js
index 2fbfbca..667985b 100644
--- a/test/simple/test-stream2-large-read-stall.js
+++ b/test/simple/test-stream2-large-read-stall.js
@@ -30,7 +30,7 @@ var PUSHSIZE = 20;
var PUSHCOUNT = 1000;
var HWM = 50;
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var r = new Readable({
highWaterMark: HWM
});
@@ -39,23 +39,23 @@ var rs = r._readableState;
r._read = push;
r.on('readable', function() {
- console.error('>> readable');
+ //console.error('>> readable');
do {
- console.error(' > read(%d)', READSIZE);
+ //console.error(' > read(%d)', READSIZE);
var ret = r.read(READSIZE);
- console.error(' < %j (%d remain)', ret && ret.length, rs.length);
+ //console.error(' < %j (%d remain)', ret && ret.length, rs.length);
} while (ret && ret.length === READSIZE);
- console.error('<< after read()',
- ret && ret.length,
- rs.needReadable,
- rs.length);
+ //console.error('<< after read()',
+ // ret && ret.length,
+ // rs.needReadable,
+ // rs.length);
});
var endEmitted = false;
r.on('end', function() {
endEmitted = true;
- console.error('end');
+ //console.error('end');
});
var pushes = 0;
@@ -64,11 +64,11 @@ function push() {
return;
if (pushes++ === PUSHCOUNT) {
- console.error(' push(EOF)');
+ //console.error(' push(EOF)');
return r.push(null);
}
- console.error(' push #%d', pushes);
+ //console.error(' push #%d', pushes);
if (r.push(new Buffer(PUSHSIZE)))
setTimeout(push);
}
diff --git a/test/simple/test-stream2-objects.js b/test/simple/test-stream2-objects.js
index 3e6931d..ff47d89 100644
--- a/test/simple/test-stream2-objects.js
+++ b/test/simple/test-stream2-objects.js
@@ -21,8 +21,8 @@
var common = require('../common.js');
-var Readable = require('_stream_readable');
-var Writable = require('_stream_writable');
+var Readable = require('../../lib/_stream_readable');
+var Writable = require('../../lib/_stream_writable');
var assert = require('assert');
// tiny node-tap lookalike.
diff --git a/test/simple/test-stream2-pipe-error-handling.js b/test/simple/test-stream2-pipe-error-handling.js
index cf7531c..e3f3e4e 100644
--- a/test/simple/test-stream2-pipe-error-handling.js
+++ b/test/simple/test-stream2-pipe-error-handling.js
@@ -21,7 +21,7 @@
var common = require('../common');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
(function testErrorListenerCatches() {
var count = 1000;
diff --git a/test/simple/test-stream2-pipe-error-once-listener.js b/test/simple/test-stream2-pipe-error-once-listener.js
index 5e8e3cb..53b2616 100755
--- a/test/simple/test-stream2-pipe-error-once-listener.js
+++ b/test/simple/test-stream2-pipe-error-once-listener.js
@@ -24,7 +24,7 @@ var common = require('../common.js');
var assert = require('assert');
var util = require('util');
-var stream = require('stream');
+var stream = require('../../');
var Read = function() {
diff --git a/test/simple/test-stream2-push.js b/test/simple/test-stream2-push.js
index b63edc3..eb2b0e9 100644
--- a/test/simple/test-stream2-push.js
+++ b/test/simple/test-stream2-push.js
@@ -20,7 +20,7 @@
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var common = require('../common.js');
-var stream = require('stream');
+var stream = require('../../');
var Readable = stream.Readable;
var Writable = stream.Writable;
var assert = require('assert');
diff --git a/test/simple/test-stream2-read-sync-stack.js b/test/simple/test-stream2-read-sync-stack.js
index e8a7305..9740a47 100644
--- a/test/simple/test-stream2-read-sync-stack.js
+++ b/test/simple/test-stream2-read-sync-stack.js
@@ -21,7 +21,7 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
var r = new Readable();
var N = 256 * 1024;
diff --git a/test/simple/test-stream2-readable-empty-buffer-no-eof.js b/test/simple/test-stream2-readable-empty-buffer-no-eof.js
index cd30178..4b1659d 100644
--- a/test/simple/test-stream2-readable-empty-buffer-no-eof.js
+++ b/test/simple/test-stream2-readable-empty-buffer-no-eof.js
@@ -22,10 +22,9 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('stream').Readable;
+var Readable = require('../../').Readable;
test1();
-test2();
function test1() {
var r = new Readable();
@@ -88,31 +87,3 @@ function test1() {
console.log('ok');
});
}
-
-function test2() {
- var r = new Readable({ encoding: 'base64' });
- var reads = 5;
- r._read = function(n) {
- if (!reads--)
- return r.push(null); // EOF
- else
- return r.push(new Buffer('x'));
- };
-
- var results = [];
- function flow() {
- var chunk;
- while (null !== (chunk = r.read()))
- results.push(chunk + '');
- }
- r.on('readable', flow);
- r.on('end', function() {
- results.push('EOF');
- });
- flow();
-
- process.on('exit', function() {
- assert.deepEqual(results, [ 'eHh4', 'eHg=', 'EOF' ]);
- console.log('ok');
- });
-}
diff --git a/test/simple/test-stream2-readable-from-list.js b/test/simple/test-stream2-readable-from-list.js
index 7c96ffe..04a96f5 100644
--- a/test/simple/test-stream2-readable-from-list.js
+++ b/test/simple/test-stream2-readable-from-list.js
@@ -21,7 +21,7 @@
var assert = require('assert');
var common = require('../common.js');
-var fromList = require('_stream_readable')._fromList;
+var fromList = require('../../lib/_stream_readable')._fromList;
// tiny node-tap lookalike.
var tests = [];
diff --git a/test/simple/test-stream2-readable-legacy-drain.js b/test/simple/test-stream2-readable-legacy-drain.js
index 675da8e..51fd3d5 100644
--- a/test/simple/test-stream2-readable-legacy-drain.js
+++ b/test/simple/test-stream2-readable-legacy-drain.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var Stream = require('stream');
+var Stream = require('../../');
var Readable = Stream.Readable;
var r = new Readable();
diff --git a/test/simple/test-stream2-readable-non-empty-end.js b/test/simple/test-stream2-readable-non-empty-end.js
index 7314ae7..c971898 100644
--- a/test/simple/test-stream2-readable-non-empty-end.js
+++ b/test/simple/test-stream2-readable-non-empty-end.js
@@ -21,7 +21,7 @@
var assert = require('assert');
var common = require('../common.js');
-var Readable = require('_stream_readable');
+var Readable = require('../../lib/_stream_readable');
var len = 0;
var chunks = new Array(10);
diff --git a/test/simple/test-stream2-readable-wrap-empty.js b/test/simple/test-stream2-readable-wrap-empty.js
index 2e5cf25..fd8a3dc 100644
--- a/test/simple/test-stream2-readable-wrap-empty.js
+++ b/test/simple/test-stream2-readable-wrap-empty.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('_stream_readable');
+var Readable = require('../../lib/_stream_readable');
var EE = require('events').EventEmitter;
var oldStream = new EE();
diff --git a/test/simple/test-stream2-readable-wrap.js b/test/simple/test-stream2-readable-wrap.js
index 90eea01..6b177f7 100644
--- a/test/simple/test-stream2-readable-wrap.js
+++ b/test/simple/test-stream2-readable-wrap.js
@@ -22,8 +22,8 @@
var common = require('../common');
var assert = require('assert');
-var Readable = require('_stream_readable');
-var Writable = require('_stream_writable');
+var Readable = require('../../lib/_stream_readable');
+var Writable = require('../../lib/_stream_writable');
var EE = require('events').EventEmitter;
var testRuns = 0, completedRuns = 0;
diff --git a/test/simple/test-stream2-set-encoding.js b/test/simple/test-stream2-set-encoding.js
index 5d2c32a..685531b 100644
--- a/test/simple/test-stream2-set-encoding.js
+++ b/test/simple/test-stream2-set-encoding.js
@@ -22,7 +22,7 @@
var common = require('../common.js');
var assert = require('assert');
-var R = require('_stream_readable');
+var R = require('../../lib/_stream_readable');
var util = require('util');
// tiny node-tap lookalike.
diff --git a/test/simple/test-stream2-transform.js b/test/simple/test-stream2-transform.js
index 9c9ddd8..a0cacc6 100644
--- a/test/simple/test-stream2-transform.js
+++ b/test/simple/test-stream2-transform.js
@@ -21,8 +21,8 @@
var assert = require('assert');
var common = require('../common.js');
-var PassThrough = require('_stream_passthrough');
-var Transform = require('_stream_transform');
+var PassThrough = require('../../').PassThrough;
+var Transform = require('../../').Transform;
// tiny node-tap lookalike.
var tests = [];
diff --git a/test/simple/test-stream2-unpipe-drain.js b/test/simple/test-stream2-unpipe-drain.js
index d66dc3c..365b327 100644
--- a/test/simple/test-stream2-unpipe-drain.js
+++ b/test/simple/test-stream2-unpipe-drain.js
@@ -22,7 +22,7 @@
var common = require('../common.js');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
var crypto = require('crypto');
var util = require('util');
diff --git a/test/simple/test-stream2-unpipe-leak.js b/test/simple/test-stream2-unpipe-leak.js
index 99f8746..17c92ae 100644
--- a/test/simple/test-stream2-unpipe-leak.js
+++ b/test/simple/test-stream2-unpipe-leak.js
@@ -22,7 +22,7 @@
var common = require('../common.js');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
var chunk = new Buffer('hallo');
diff --git a/test/simple/test-stream2-writable.js b/test/simple/test-stream2-writable.js
index 704100c..209c3a6 100644
--- a/test/simple/test-stream2-writable.js
+++ b/test/simple/test-stream2-writable.js
@@ -20,8 +20,8 @@
// USE OR OTHER DEALINGS IN THE SOFTWARE.
var common = require('../common.js');
-var W = require('_stream_writable');
-var D = require('_stream_duplex');
+var W = require('../../').Writable;
+var D = require('../../').Duplex;
var assert = require('assert');
var util = require('util');
diff --git a/test/simple/test-stream3-pause-then-read.js b/test/simple/test-stream3-pause-then-read.js
index b91bde3..2f72c15 100644
--- a/test/simple/test-stream3-pause-then-read.js
+++ b/test/simple/test-stream3-pause-then-read.js
@@ -22,7 +22,7 @@
var common = require('../common');
var assert = require('assert');
-var stream = require('stream');
+var stream = require('../../');
var Readable = stream.Readable;
var Writable = stream.Writable;

View File

@@ -0,0 +1,89 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
// a duplex stream is just a stream that is both readable and writable.
// Since JS doesn't have multiple prototypal inheritance, this class
// prototypally inherits from Readable, and then parasitically from
// Writable.
module.exports = Duplex;
/*<replacement>*/
var objectKeys = Object.keys || function (obj) {
var keys = [];
for (var key in obj) keys.push(key);
return keys;
}
/*</replacement>*/
/*<replacement>*/
var util = require('core-util-is');
util.inherits = require('inherits');
/*</replacement>*/
var Readable = require('./_stream_readable');
var Writable = require('./_stream_writable');
util.inherits(Duplex, Readable);
forEach(objectKeys(Writable.prototype), function(method) {
if (!Duplex.prototype[method])
Duplex.prototype[method] = Writable.prototype[method];
});
function Duplex(options) {
if (!(this instanceof Duplex))
return new Duplex(options);
Readable.call(this, options);
Writable.call(this, options);
if (options && options.readable === false)
this.readable = false;
if (options && options.writable === false)
this.writable = false;
this.allowHalfOpen = true;
if (options && options.allowHalfOpen === false)
this.allowHalfOpen = false;
this.once('end', onend);
}
// the no-half-open enforcer
function onend() {
// if we allow half-open state, or if the writable side ended,
// then we're ok.
if (this.allowHalfOpen || this._writableState.ended)
return;
// no more data can be written.
// But allow more writes to happen in this tick.
process.nextTick(this.end.bind(this));
}
function forEach (xs, f) {
for (var i = 0, l = xs.length; i < l; i++) {
f(xs[i], i);
}
}

View File

@@ -0,0 +1,46 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
// a passthrough stream.
// basically just the most minimal sort of Transform stream.
// Every written chunk gets output as-is.
module.exports = PassThrough;
var Transform = require('./_stream_transform');
/*<replacement>*/
var util = require('core-util-is');
util.inherits = require('inherits');
/*</replacement>*/
util.inherits(PassThrough, Transform);
function PassThrough(options) {
if (!(this instanceof PassThrough))
return new PassThrough(options);
Transform.call(this, options);
}
PassThrough.prototype._transform = function(chunk, encoding, cb) {
cb(null, chunk);
};

View File

@@ -0,0 +1,951 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
module.exports = Readable;
/*<replacement>*/
var isArray = require('isarray');
/*</replacement>*/
/*<replacement>*/
var Buffer = require('buffer').Buffer;
/*</replacement>*/
Readable.ReadableState = ReadableState;
var EE = require('events').EventEmitter;
/*<replacement>*/
if (!EE.listenerCount) EE.listenerCount = function(emitter, type) {
return emitter.listeners(type).length;
};
/*</replacement>*/
var Stream = require('stream');
/*<replacement>*/
var util = require('core-util-is');
util.inherits = require('inherits');
/*</replacement>*/
var StringDecoder;
/*<replacement>*/
var debug = require('util');
if (debug && debug.debuglog) {
debug = debug.debuglog('stream');
} else {
debug = function () {};
}
/*</replacement>*/
util.inherits(Readable, Stream);
function ReadableState(options, stream) {
var Duplex = require('./_stream_duplex');
options = options || {};
// the point at which it stops calling _read() to fill the buffer
// Note: 0 is a valid value, means "don't call _read preemptively ever"
var hwm = options.highWaterMark;
var defaultHwm = options.objectMode ? 16 : 16 * 1024;
this.highWaterMark = (hwm || hwm === 0) ? hwm : defaultHwm;
// cast to ints.
this.highWaterMark = ~~this.highWaterMark;
this.buffer = [];
this.length = 0;
this.pipes = null;
this.pipesCount = 0;
this.flowing = null;
this.ended = false;
this.endEmitted = false;
this.reading = false;
// a flag to be able to tell if the onwrite cb is called immediately,
// or on a later tick. We set this to true at first, because any
// actions that shouldn't happen until "later" should generally also
// not happen before the first write call.
this.sync = true;
// whenever we return null, then we set a flag to say
// that we're awaiting a 'readable' event emission.
this.needReadable = false;
this.emittedReadable = false;
this.readableListening = false;
// object stream flag. Used to make read(n) ignore n and to
// make all the buffer merging and length checks go away
this.objectMode = !!options.objectMode;
if (stream instanceof Duplex)
this.objectMode = this.objectMode || !!options.readableObjectMode;
// Crypto is kind of old and crusty. Historically, its default string
// encoding is 'binary' so we have to make this configurable.
// Everything else in the universe uses 'utf8', though.
this.defaultEncoding = options.defaultEncoding || 'utf8';
// when piping, we only care about 'readable' events that happen
// after read()ing all the bytes and not getting any pushback.
this.ranOut = false;
// the number of writers that are awaiting a drain event in .pipe()s
this.awaitDrain = 0;
// if true, a maybeReadMore has been scheduled
this.readingMore = false;
this.decoder = null;
this.encoding = null;
if (options.encoding) {
if (!StringDecoder)
StringDecoder = require('string_decoder/').StringDecoder;
this.decoder = new StringDecoder(options.encoding);
this.encoding = options.encoding;
}
}
function Readable(options) {
var Duplex = require('./_stream_duplex');
if (!(this instanceof Readable))
return new Readable(options);
this._readableState = new ReadableState(options, this);
// legacy
this.readable = true;
Stream.call(this);
}
// Manually shove something into the read() buffer.
// This returns true if the highWaterMark has not been hit yet,
// similar to how Writable.write() returns true if you should
// write() some more.
Readable.prototype.push = function(chunk, encoding) {
var state = this._readableState;
if (util.isString(chunk) && !state.objectMode) {
encoding = encoding || state.defaultEncoding;
if (encoding !== state.encoding) {
chunk = new Buffer(chunk, encoding);
encoding = '';
}
}
return readableAddChunk(this, state, chunk, encoding, false);
};
// Unshift should *always* be something directly out of read()
Readable.prototype.unshift = function(chunk) {
var state = this._readableState;
return readableAddChunk(this, state, chunk, '', true);
};
function readableAddChunk(stream, state, chunk, encoding, addToFront) {
var er = chunkInvalid(state, chunk);
if (er) {
stream.emit('error', er);
} else if (util.isNullOrUndefined(chunk)) {
state.reading = false;
if (!state.ended)
onEofChunk(stream, state);
} else if (state.objectMode || chunk && chunk.length > 0) {
if (state.ended && !addToFront) {
var e = new Error('stream.push() after EOF');
stream.emit('error', e);
} else if (state.endEmitted && addToFront) {
var e = new Error('stream.unshift() after end event');
stream.emit('error', e);
} else {
if (state.decoder && !addToFront && !encoding)
chunk = state.decoder.write(chunk);
if (!addToFront)
state.reading = false;
// if we want the data now, just emit it.
if (state.flowing && state.length === 0 && !state.sync) {
stream.emit('data', chunk);
stream.read(0);
} else {
// update the buffer info.
state.length += state.objectMode ? 1 : chunk.length;
if (addToFront)
state.buffer.unshift(chunk);
else
state.buffer.push(chunk);
if (state.needReadable)
emitReadable(stream);
}
maybeReadMore(stream, state);
}
} else if (!addToFront) {
state.reading = false;
}
return needMoreData(state);
}
// if it's past the high water mark, we can push in some more.
// Also, if we have no data yet, we can stand some
// more bytes. This is to work around cases where hwm=0,
// such as the repl. Also, if the push() triggered a
// readable event, and the user called read(largeNumber) such that
// needReadable was set, then we ought to push more, so that another
// 'readable' event will be triggered.
function needMoreData(state) {
return !state.ended &&
(state.needReadable ||
state.length < state.highWaterMark ||
state.length === 0);
}
// backwards compatibility.
Readable.prototype.setEncoding = function(enc) {
if (!StringDecoder)
StringDecoder = require('string_decoder/').StringDecoder;
this._readableState.decoder = new StringDecoder(enc);
this._readableState.encoding = enc;
return this;
};
// Don't raise the hwm > 128MB
var MAX_HWM = 0x800000;
function roundUpToNextPowerOf2(n) {
if (n >= MAX_HWM) {
n = MAX_HWM;
} else {
// Get the next highest power of 2
n--;
for (var p = 1; p < 32; p <<= 1) n |= n >> p;
n++;
}
return n;
}
function howMuchToRead(n, state) {
if (state.length === 0 && state.ended)
return 0;
if (state.objectMode)
return n === 0 ? 0 : 1;
if (isNaN(n) || util.isNull(n)) {
// only flow one buffer at a time
if (state.flowing && state.buffer.length)
return state.buffer[0].length;
else
return state.length;
}
if (n <= 0)
return 0;
// If we're asking for more than the target buffer level,
// then raise the water mark. Bump up to the next highest
// power of 2, to prevent increasing it excessively in tiny
// amounts.
if (n > state.highWaterMark)
state.highWaterMark = roundUpToNextPowerOf2(n);
// don't have that much. return null, unless we've ended.
if (n > state.length) {
if (!state.ended) {
state.needReadable = true;
return 0;
} else
return state.length;
}
return n;
}
// you can override either this method, or the async _read(n) below.
Readable.prototype.read = function(n) {
debug('read', n);
var state = this._readableState;
var nOrig = n;
if (!util.isNumber(n) || n > 0)
state.emittedReadable = false;
// if we're doing read(0) to trigger a readable event, but we
// already have a bunch of data in the buffer, then just trigger
// the 'readable' event and move on.
if (n === 0 &&
state.needReadable &&
(state.length >= state.highWaterMark || state.ended)) {
debug('read: emitReadable', state.length, state.ended);
if (state.length === 0 && state.ended)
endReadable(this);
else
emitReadable(this);
return null;
}
n = howMuchToRead(n, state);
// if we've ended, and we're now clear, then finish it up.
if (n === 0 && state.ended) {
if (state.length === 0)
endReadable(this);
return null;
}
// All the actual chunk generation logic needs to be
// *below* the call to _read. The reason is that in certain
// synthetic stream cases, such as passthrough streams, _read
// may be a completely synchronous operation which may change
// the state of the read buffer, providing enough data when
// before there was *not* enough.
//
// So, the steps are:
// 1. Figure out what the state of things will be after we do
// a read from the buffer.
//
// 2. If that resulting state will trigger a _read, then call _read.
// Note that this may be asynchronous, or synchronous. Yes, it is
// deeply ugly to write APIs this way, but that still doesn't mean
// that the Readable class should behave improperly, as streams are
// designed to be sync/async agnostic.
// Take note if the _read call is sync or async (ie, if the read call
// has returned yet), so that we know whether or not it's safe to emit
// 'readable' etc.
//
// 3. Actually pull the requested chunks out of the buffer and return.
// if we need a readable event, then we need to do some reading.
var doRead = state.needReadable;
debug('need readable', doRead);
// if we currently have less than the highWaterMark, then also read some
if (state.length === 0 || state.length - n < state.highWaterMark) {
doRead = true;
debug('length less than watermark', doRead);
}
// however, if we've ended, then there's no point, and if we're already
// reading, then it's unnecessary.
if (state.ended || state.reading) {
doRead = false;
debug('reading or ended', doRead);
}
if (doRead) {
debug('do read');
state.reading = true;
state.sync = true;
// if the length is currently zero, then we *need* a readable event.
if (state.length === 0)
state.needReadable = true;
// call internal read method
this._read(state.highWaterMark);
state.sync = false;
}
// If _read pushed data synchronously, then `reading` will be false,
// and we need to re-evaluate how much data we can return to the user.
if (doRead && !state.reading)
n = howMuchToRead(nOrig, state);
var ret;
if (n > 0)
ret = fromList(n, state);
else
ret = null;
if (util.isNull(ret)) {
state.needReadable = true;
n = 0;
}
state.length -= n;
// If we have nothing in the buffer, then we want to know
// as soon as we *do* get something into the buffer.
if (state.length === 0 && !state.ended)
state.needReadable = true;
// If we tried to read() past the EOF, then emit end on the next tick.
if (nOrig !== n && state.ended && state.length === 0)
endReadable(this);
if (!util.isNull(ret))
this.emit('data', ret);
return ret;
};
function chunkInvalid(state, chunk) {
var er = null;
if (!util.isBuffer(chunk) &&
!util.isString(chunk) &&
!util.isNullOrUndefined(chunk) &&
!state.objectMode) {
er = new TypeError('Invalid non-string/buffer chunk');
}
return er;
}
function onEofChunk(stream, state) {
if (state.decoder && !state.ended) {
var chunk = state.decoder.end();
if (chunk && chunk.length) {
state.buffer.push(chunk);
state.length += state.objectMode ? 1 : chunk.length;
}
}
state.ended = true;
// emit 'readable' now to make sure it gets picked up.
emitReadable(stream);
}
// Don't emit readable right away in sync mode, because this can trigger
// another read() call => stack overflow. This way, it might trigger
// a nextTick recursion warning, but that's not so bad.
function emitReadable(stream) {
var state = stream._readableState;
state.needReadable = false;
if (!state.emittedReadable) {
debug('emitReadable', state.flowing);
state.emittedReadable = true;
if (state.sync)
process.nextTick(function() {
emitReadable_(stream);
});
else
emitReadable_(stream);
}
}
function emitReadable_(stream) {
debug('emit readable');
stream.emit('readable');
flow(stream);
}
// at this point, the user has presumably seen the 'readable' event,
// and called read() to consume some data. that may have triggered
// in turn another _read(n) call, in which case reading = true if
// it's in progress.
// However, if we're not ended, or reading, and the length < hwm,
// then go ahead and try to read some more preemptively.
function maybeReadMore(stream, state) {
if (!state.readingMore) {
state.readingMore = true;
process.nextTick(function() {
maybeReadMore_(stream, state);
});
}
}
function maybeReadMore_(stream, state) {
var len = state.length;
while (!state.reading && !state.flowing && !state.ended &&
state.length < state.highWaterMark) {
debug('maybeReadMore read 0');
stream.read(0);
if (len === state.length)
// didn't get any data, stop spinning.
break;
else
len = state.length;
}
state.readingMore = false;
}
// abstract method. to be overridden in specific implementation classes.
// call cb(er, data) where data is <= n in length.
// for virtual (non-string, non-buffer) streams, "length" is somewhat
// arbitrary, and perhaps not very meaningful.
Readable.prototype._read = function(n) {
this.emit('error', new Error('not implemented'));
};
Readable.prototype.pipe = function(dest, pipeOpts) {
var src = this;
var state = this._readableState;
switch (state.pipesCount) {
case 0:
state.pipes = dest;
break;
case 1:
state.pipes = [state.pipes, dest];
break;
default:
state.pipes.push(dest);
break;
}
state.pipesCount += 1;
debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts);
var doEnd = (!pipeOpts || pipeOpts.end !== false) &&
dest !== process.stdout &&
dest !== process.stderr;
var endFn = doEnd ? onend : cleanup;
if (state.endEmitted)
process.nextTick(endFn);
else
src.once('end', endFn);
dest.on('unpipe', onunpipe);
function onunpipe(readable) {
debug('onunpipe');
if (readable === src) {
cleanup();
}
}
function onend() {
debug('onend');
dest.end();
}
// when the dest drains, it reduces the awaitDrain counter
// on the source. This would be more elegant with a .once()
// handler in flow(), but adding and removing repeatedly is
// too slow.
var ondrain = pipeOnDrain(src);
dest.on('drain', ondrain);
function cleanup() {
debug('cleanup');
// cleanup event handlers once the pipe is broken
dest.removeListener('close', onclose);
dest.removeListener('finish', onfinish);
dest.removeListener('drain', ondrain);
dest.removeListener('error', onerror);
dest.removeListener('unpipe', onunpipe);
src.removeListener('end', onend);
src.removeListener('end', cleanup);
src.removeListener('data', ondata);
// if the reader is waiting for a drain event from this
// specific writer, then it would cause it to never start
// flowing again.
// So, if this is awaiting a drain, then we just call it now.
// If we don't know, then assume that we are waiting for one.
if (state.awaitDrain &&
(!dest._writableState || dest._writableState.needDrain))
ondrain();
}
src.on('data', ondata);
function ondata(chunk) {
debug('ondata');
var ret = dest.write(chunk);
if (false === ret) {
debug('false write response, pause',
src._readableState.awaitDrain);
src._readableState.awaitDrain++;
src.pause();
}
}
// if the dest has an error, then stop piping into it.
// however, don't suppress the throwing behavior for this.
function onerror(er) {
debug('onerror', er);
unpipe();
dest.removeListener('error', onerror);
if (EE.listenerCount(dest, 'error') === 0)
dest.emit('error', er);
}
// This is a brutally ugly hack to make sure that our error handler
// is attached before any userland ones. NEVER DO THIS.
if (!dest._events || !dest._events.error)
dest.on('error', onerror);
else if (isArray(dest._events.error))
dest._events.error.unshift(onerror);
else
dest._events.error = [onerror, dest._events.error];
// Both close and finish should trigger unpipe, but only once.
function onclose() {
dest.removeListener('finish', onfinish);
unpipe();
}
dest.once('close', onclose);
function onfinish() {
debug('onfinish');
dest.removeListener('close', onclose);
unpipe();
}
dest.once('finish', onfinish);
function unpipe() {
debug('unpipe');
src.unpipe(dest);
}
// tell the dest that it's being piped to
dest.emit('pipe', src);
// start the flow if it hasn't been started already.
if (!state.flowing) {
debug('pipe resume');
src.resume();
}
return dest;
};
function pipeOnDrain(src) {
return function() {
var state = src._readableState;
debug('pipeOnDrain', state.awaitDrain);
if (state.awaitDrain)
state.awaitDrain--;
if (state.awaitDrain === 0 && EE.listenerCount(src, 'data')) {
state.flowing = true;
flow(src);
}
};
}
Readable.prototype.unpipe = function(dest) {
var state = this._readableState;
// if we're not piping anywhere, then do nothing.
if (state.pipesCount === 0)
return this;
// just one destination. most common case.
if (state.pipesCount === 1) {
// passed in one, but it's not the right one.
if (dest && dest !== state.pipes)
return this;
if (!dest)
dest = state.pipes;
// got a match.
state.pipes = null;
state.pipesCount = 0;
state.flowing = false;
if (dest)
dest.emit('unpipe', this);
return this;
}
// slow case. multiple pipe destinations.
if (!dest) {
// remove all.
var dests = state.pipes;
var len = state.pipesCount;
state.pipes = null;
state.pipesCount = 0;
state.flowing = false;
for (var i = 0; i < len; i++)
dests[i].emit('unpipe', this);
return this;
}
// try to find the right one.
var i = indexOf(state.pipes, dest);
if (i === -1)
return this;
state.pipes.splice(i, 1);
state.pipesCount -= 1;
if (state.pipesCount === 1)
state.pipes = state.pipes[0];
dest.emit('unpipe', this);
return this;
};
// set up data events if they are asked for
// Ensure readable listeners eventually get something
Readable.prototype.on = function(ev, fn) {
var res = Stream.prototype.on.call(this, ev, fn);
// If listening to data, and it has not explicitly been paused,
// then call resume to start the flow of data on the next tick.
if (ev === 'data' && false !== this._readableState.flowing) {
this.resume();
}
if (ev === 'readable' && this.readable) {
var state = this._readableState;
if (!state.readableListening) {
state.readableListening = true;
state.emittedReadable = false;
state.needReadable = true;
if (!state.reading) {
var self = this;
process.nextTick(function() {
debug('readable nexttick read 0');
self.read(0);
});
} else if (state.length) {
emitReadable(this, state);
}
}
}
return res;
};
Readable.prototype.addListener = Readable.prototype.on;
// pause() and resume() are remnants of the legacy readable stream API
// If the user uses them, then switch into old mode.
Readable.prototype.resume = function() {
var state = this._readableState;
if (!state.flowing) {
debug('resume');
state.flowing = true;
if (!state.reading) {
debug('resume read 0');
this.read(0);
}
resume(this, state);
}
return this;
};
function resume(stream, state) {
if (!state.resumeScheduled) {
state.resumeScheduled = true;
process.nextTick(function() {
resume_(stream, state);
});
}
}
function resume_(stream, state) {
state.resumeScheduled = false;
stream.emit('resume');
flow(stream);
if (state.flowing && !state.reading)
stream.read(0);
}
Readable.prototype.pause = function() {
debug('call pause flowing=%j', this._readableState.flowing);
if (false !== this._readableState.flowing) {
debug('pause');
this._readableState.flowing = false;
this.emit('pause');
}
return this;
};
function flow(stream) {
var state = stream._readableState;
debug('flow', state.flowing);
if (state.flowing) {
do {
var chunk = stream.read();
} while (null !== chunk && state.flowing);
}
}
// wrap an old-style stream as the async data source.
// This is *not* part of the readable stream interface.
// It is an ugly unfortunate mess of history.
Readable.prototype.wrap = function(stream) {
var state = this._readableState;
var paused = false;
var self = this;
stream.on('end', function() {
debug('wrapped end');
if (state.decoder && !state.ended) {
var chunk = state.decoder.end();
if (chunk && chunk.length)
self.push(chunk);
}
self.push(null);
});
stream.on('data', function(chunk) {
debug('wrapped data');
if (state.decoder)
chunk = state.decoder.write(chunk);
if (!chunk || !state.objectMode && !chunk.length)
return;
var ret = self.push(chunk);
if (!ret) {
paused = true;
stream.pause();
}
});
// proxy all the other methods.
// important when wrapping filters and duplexes.
for (var i in stream) {
if (util.isFunction(stream[i]) && util.isUndefined(this[i])) {
this[i] = function(method) { return function() {
return stream[method].apply(stream, arguments);
}}(i);
}
}
// proxy certain important events.
var events = ['error', 'close', 'destroy', 'pause', 'resume'];
forEach(events, function(ev) {
stream.on(ev, self.emit.bind(self, ev));
});
// when we try to consume some more bytes, simply unpause the
// underlying stream.
self._read = function(n) {
debug('wrapped _read', n);
if (paused) {
paused = false;
stream.resume();
}
};
return self;
};
// exposed for testing purposes only.
Readable._fromList = fromList;
// Pluck off n bytes from an array of buffers.
// Length is the combined lengths of all the buffers in the list.
function fromList(n, state) {
var list = state.buffer;
var length = state.length;
var stringMode = !!state.decoder;
var objectMode = !!state.objectMode;
var ret;
// nothing in the list, definitely empty.
if (list.length === 0)
return null;
if (length === 0)
ret = null;
else if (objectMode)
ret = list.shift();
else if (!n || n >= length) {
// read it all, truncate the array.
if (stringMode)
ret = list.join('');
else
ret = Buffer.concat(list, length);
list.length = 0;
} else {
// read just some of it.
if (n < list[0].length) {
// just take a part of the first list item.
// slice is the same for buffers and strings.
var buf = list[0];
ret = buf.slice(0, n);
list[0] = buf.slice(n);
} else if (n === list[0].length) {
// first list is a perfect match
ret = list.shift();
} else {
// complex case.
// we have enough to cover it, but it spans past the first buffer.
if (stringMode)
ret = '';
else
ret = new Buffer(n);
var c = 0;
for (var i = 0, l = list.length; i < l && c < n; i++) {
var buf = list[0];
var cpy = Math.min(n - c, buf.length);
if (stringMode)
ret += buf.slice(0, cpy);
else
buf.copy(ret, c, 0, cpy);
if (cpy < buf.length)
list[0] = buf.slice(cpy);
else
list.shift();
c += cpy;
}
}
}
return ret;
}
function endReadable(stream) {
var state = stream._readableState;
// If we get here before consuming all the bytes, then that is a
// bug in node. Should never happen.
if (state.length > 0)
throw new Error('endReadable called on non-empty stream');
if (!state.endEmitted) {
state.ended = true;
process.nextTick(function() {
// Check that we didn't get one last unshift.
if (!state.endEmitted && state.length === 0) {
state.endEmitted = true;
stream.readable = false;
stream.emit('end');
}
});
}
}
function forEach (xs, f) {
for (var i = 0, l = xs.length; i < l; i++) {
f(xs[i], i);
}
}
function indexOf (xs, x) {
for (var i = 0, l = xs.length; i < l; i++) {
if (xs[i] === x) return i;
}
return -1;
}

View File

@@ -0,0 +1,209 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
// a transform stream is a readable/writable stream where you do
// something with the data. Sometimes it's called a "filter",
// but that's not a great name for it, since that implies a thing where
// some bits pass through, and others are simply ignored. (That would
// be a valid example of a transform, of course.)
//
// While the output is causally related to the input, it's not a
// necessarily symmetric or synchronous transformation. For example,
// a zlib stream might take multiple plain-text writes(), and then
// emit a single compressed chunk some time in the future.
//
// Here's how this works:
//
// The Transform stream has all the aspects of the readable and writable
// stream classes. When you write(chunk), that calls _write(chunk,cb)
// internally, and returns false if there's a lot of pending writes
// buffered up. When you call read(), that calls _read(n) until
// there's enough pending readable data buffered up.
//
// In a transform stream, the written data is placed in a buffer. When
// _read(n) is called, it transforms the queued up data, calling the
// buffered _write cb's as it consumes chunks. If consuming a single
// written chunk would result in multiple output chunks, then the first
// outputted bit calls the readcb, and subsequent chunks just go into
// the read buffer, and will cause it to emit 'readable' if necessary.
//
// This way, back-pressure is actually determined by the reading side,
// since _read has to be called to start processing a new chunk. However,
// a pathological inflate type of transform can cause excessive buffering
// here. For example, imagine a stream where every byte of input is
// interpreted as an integer from 0-255, and then results in that many
// bytes of output. Writing the 4 bytes {ff,ff,ff,ff} would result in
// 1kb of data being output. In this case, you could write a very small
// amount of input, and end up with a very large amount of output. In
// such a pathological inflating mechanism, there'd be no way to tell
// the system to stop doing the transform. A single 4MB write could
// cause the system to run out of memory.
//
// However, even in such a pathological case, only a single written chunk
// would be consumed, and then the rest would wait (un-transformed) until
// the results of the previous transformed chunk were consumed.
module.exports = Transform;
var Duplex = require('./_stream_duplex');
/*<replacement>*/
var util = require('core-util-is');
util.inherits = require('inherits');
/*</replacement>*/
util.inherits(Transform, Duplex);
function TransformState(options, stream) {
this.afterTransform = function(er, data) {
return afterTransform(stream, er, data);
};
this.needTransform = false;
this.transforming = false;
this.writecb = null;
this.writechunk = null;
}
function afterTransform(stream, er, data) {
var ts = stream._transformState;
ts.transforming = false;
var cb = ts.writecb;
if (!cb)
return stream.emit('error', new Error('no writecb in Transform class'));
ts.writechunk = null;
ts.writecb = null;
if (!util.isNullOrUndefined(data))
stream.push(data);
if (cb)
cb(er);
var rs = stream._readableState;
rs.reading = false;
if (rs.needReadable || rs.length < rs.highWaterMark) {
stream._read(rs.highWaterMark);
}
}
function Transform(options) {
if (!(this instanceof Transform))
return new Transform(options);
Duplex.call(this, options);
this._transformState = new TransformState(options, this);
// when the writable side finishes, then flush out anything remaining.
var stream = this;
// start out asking for a readable event once data is transformed.
this._readableState.needReadable = true;
// we have implemented the _read method, and done the other things
// that Readable wants before the first _read call, so unset the
// sync guard flag.
this._readableState.sync = false;
this.once('prefinish', function() {
if (util.isFunction(this._flush))
this._flush(function(er) {
done(stream, er);
});
else
done(stream);
});
}
Transform.prototype.push = function(chunk, encoding) {
this._transformState.needTransform = false;
return Duplex.prototype.push.call(this, chunk, encoding);
};
// This is the part where you do stuff!
// override this function in implementation classes.
// 'chunk' is an input chunk.
//
// Call `push(newChunk)` to pass along transformed output
// to the readable side. You may call 'push' zero or more times.
//
// Call `cb(err)` when you are done with this chunk. If you pass
// an error, then that'll put the hurt on the whole operation. If you
// never call cb(), then you'll never get another chunk.
Transform.prototype._transform = function(chunk, encoding, cb) {
throw new Error('not implemented');
};
Transform.prototype._write = function(chunk, encoding, cb) {
var ts = this._transformState;
ts.writecb = cb;
ts.writechunk = chunk;
ts.writeencoding = encoding;
if (!ts.transforming) {
var rs = this._readableState;
if (ts.needTransform ||
rs.needReadable ||
rs.length < rs.highWaterMark)
this._read(rs.highWaterMark);
}
};
// Doesn't matter what the args are here.
// _transform does all the work.
// That we got here means that the readable side wants more data.
Transform.prototype._read = function(n) {
var ts = this._transformState;
if (!util.isNull(ts.writechunk) && ts.writecb && !ts.transforming) {
ts.transforming = true;
this._transform(ts.writechunk, ts.writeencoding, ts.afterTransform);
} else {
// mark that we need a transform, so that any data that comes in
// will get processed, now that we've asked for it.
ts.needTransform = true;
}
};
function done(stream, er) {
if (er)
return stream.emit('error', er);
// if there's nothing in the write buffer, then that means
// that nothing more will ever be provided
var ws = stream._writableState;
var ts = stream._transformState;
if (ws.length)
throw new Error('calling transform done when ws.length != 0');
if (ts.transforming)
throw new Error('calling transform done when still transforming');
return stream.push(null);
}

View File

@@ -0,0 +1,477 @@
// Copyright Joyent, Inc. and other Node contributors.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to permit
// persons to whom the Software is furnished to do so, subject to the
// following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
// NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
// USE OR OTHER DEALINGS IN THE SOFTWARE.
// A bit simpler than readable streams.
// Implement an async ._write(chunk, cb), and it'll handle all
// the drain event emission and buffering.
module.exports = Writable;
/*<replacement>*/
var Buffer = require('buffer').Buffer;
/*</replacement>*/
Writable.WritableState = WritableState;
/*<replacement>*/
var util = require('core-util-is');
util.inherits = require('inherits');
/*</replacement>*/
var Stream = require('stream');
util.inherits(Writable, Stream);
function WriteReq(chunk, encoding, cb) {
this.chunk = chunk;
this.encoding = encoding;
this.callback = cb;
}
function WritableState(options, stream) {
var Duplex = require('./_stream_duplex');
options = options || {};
// the point at which write() starts returning false
// Note: 0 is a valid value, means that we always return false if
// the entire buffer is not flushed immediately on write()
var hwm = options.highWaterMark;
var defaultHwm = options.objectMode ? 16 : 16 * 1024;
this.highWaterMark = (hwm || hwm === 0) ? hwm : defaultHwm;
// object stream flag to indicate whether or not this stream
// contains buffers or objects.
this.objectMode = !!options.objectMode;
if (stream instanceof Duplex)
this.objectMode = this.objectMode || !!options.writableObjectMode;
// cast to ints.
this.highWaterMark = ~~this.highWaterMark;
this.needDrain = false;
// at the start of calling end()
this.ending = false;
// when end() has been called, and returned
this.ended = false;
// when 'finish' is emitted
this.finished = false;
// should we decode strings into buffers before passing to _write?
// this is here so that some node-core streams can optimize string
// handling at a lower level.
var noDecode = options.decodeStrings === false;
this.decodeStrings = !noDecode;
// Crypto is kind of old and crusty. Historically, its default string
// encoding is 'binary' so we have to make this configurable.
// Everything else in the universe uses 'utf8', though.
this.defaultEncoding = options.defaultEncoding || 'utf8';
// not an actual buffer we keep track of, but a measurement
// of how much we're waiting to get pushed to some underlying
// socket or file.
this.length = 0;
// a flag to see when we're in the middle of a write.
this.writing = false;
// when true all writes will be buffered until .uncork() call
this.corked = 0;
// a flag to be able to tell if the onwrite cb is called immediately,
// or on a later tick. We set this to true at first, because any
// actions that shouldn't happen until "later" should generally also
// not happen before the first write call.
this.sync = true;
// a flag to know if we're processing previously buffered items, which
// may call the _write() callback in the same tick, so that we don't
// end up in an overlapped onwrite situation.
this.bufferProcessing = false;
// the callback that's passed to _write(chunk,cb)
this.onwrite = function(er) {
onwrite(stream, er);
};
// the callback that the user supplies to write(chunk,encoding,cb)
this.writecb = null;
// the amount that is being written when _write is called.
this.writelen = 0;
this.buffer = [];
// number of pending user-supplied write callbacks
// this must be 0 before 'finish' can be emitted
this.pendingcb = 0;
// emit prefinish if the only thing we're waiting for is _write cbs
// This is relevant for synchronous Transform streams
this.prefinished = false;
// True if the error was already emitted and should not be thrown again
this.errorEmitted = false;
}
function Writable(options) {
var Duplex = require('./_stream_duplex');
// Writable ctor is applied to Duplexes, though they're not
// instanceof Writable, they're instanceof Readable.
if (!(this instanceof Writable) && !(this instanceof Duplex))
return new Writable(options);
this._writableState = new WritableState(options, this);
// legacy.
this.writable = true;
Stream.call(this);
}
// Otherwise people can pipe Writable streams, which is just wrong.
Writable.prototype.pipe = function() {
this.emit('error', new Error('Cannot pipe. Not readable.'));
};
function writeAfterEnd(stream, state, cb) {
var er = new Error('write after end');
// TODO: defer error events consistently everywhere, not just the cb
stream.emit('error', er);
process.nextTick(function() {
cb(er);
});
}
// If we get something that is not a buffer, string, null, or undefined,
// and we're not in objectMode, then that's an error.
// Otherwise stream chunks are all considered to be of length=1, and the
// watermarks determine how many objects to keep in the buffer, rather than
// how many bytes or characters.
function validChunk(stream, state, chunk, cb) {
var valid = true;
if (!util.isBuffer(chunk) &&
!util.isString(chunk) &&
!util.isNullOrUndefined(chunk) &&
!state.objectMode) {
var er = new TypeError('Invalid non-string/buffer chunk');
stream.emit('error', er);
process.nextTick(function() {
cb(er);
});
valid = false;
}
return valid;
}
Writable.prototype.write = function(chunk, encoding, cb) {
var state = this._writableState;
var ret = false;
if (util.isFunction(encoding)) {
cb = encoding;
encoding = null;
}
if (util.isBuffer(chunk))
encoding = 'buffer';
else if (!encoding)
encoding = state.defaultEncoding;
if (!util.isFunction(cb))
cb = function() {};
if (state.ended)
writeAfterEnd(this, state, cb);
else if (validChunk(this, state, chunk, cb)) {
state.pendingcb++;
ret = writeOrBuffer(this, state, chunk, encoding, cb);
}
return ret;
};
Writable.prototype.cork = function() {
var state = this._writableState;
state.corked++;
};
Writable.prototype.uncork = function() {
var state = this._writableState;
if (state.corked) {
state.corked--;
if (!state.writing &&
!state.corked &&
!state.finished &&
!state.bufferProcessing &&
state.buffer.length)
clearBuffer(this, state);
}
};
function decodeChunk(state, chunk, encoding) {
if (!state.objectMode &&
state.decodeStrings !== false &&
util.isString(chunk)) {
chunk = new Buffer(chunk, encoding);
}
return chunk;
}
// if we're already writing something, then just put this
// in the queue, and wait our turn. Otherwise, call _write
// If we return false, then we need a drain event, so set that flag.
function writeOrBuffer(stream, state, chunk, encoding, cb) {
chunk = decodeChunk(state, chunk, encoding);
if (util.isBuffer(chunk))
encoding = 'buffer';
var len = state.objectMode ? 1 : chunk.length;
state.length += len;
var ret = state.length < state.highWaterMark;
// we must ensure that previous needDrain will not be reset to false.
if (!ret)
state.needDrain = true;
if (state.writing || state.corked)
state.buffer.push(new WriteReq(chunk, encoding, cb));
else
doWrite(stream, state, false, len, chunk, encoding, cb);
return ret;
}
function doWrite(stream, state, writev, len, chunk, encoding, cb) {
state.writelen = len;
state.writecb = cb;
state.writing = true;
state.sync = true;
if (writev)
stream._writev(chunk, state.onwrite);
else
stream._write(chunk, encoding, state.onwrite);
state.sync = false;
}
function onwriteError(stream, state, sync, er, cb) {
if (sync)
process.nextTick(function() {
state.pendingcb--;
cb(er);
});
else {
state.pendingcb--;
cb(er);
}
stream._writableState.errorEmitted = true;
stream.emit('error', er);
}
function onwriteStateUpdate(state) {
state.writing = false;
state.writecb = null;
state.length -= state.writelen;
state.writelen = 0;
}
function onwrite(stream, er) {
var state = stream._writableState;
var sync = state.sync;
var cb = state.writecb;
onwriteStateUpdate(state);
if (er)
onwriteError(stream, state, sync, er, cb);
else {
// Check if we're actually ready to finish, but don't emit yet
var finished = needFinish(stream, state);
if (!finished &&
!state.corked &&
!state.bufferProcessing &&
state.buffer.length) {
clearBuffer(stream, state);
}
if (sync) {
process.nextTick(function() {
afterWrite(stream, state, finished, cb);
});
} else {
afterWrite(stream, state, finished, cb);
}
}
}
function afterWrite(stream, state, finished, cb) {
if (!finished)
onwriteDrain(stream, state);
state.pendingcb--;
cb();
finishMaybe(stream, state);
}
// Must force callback to be called on nextTick, so that we don't
// emit 'drain' before the write() consumer gets the 'false' return
// value, and has a chance to attach a 'drain' listener.
function onwriteDrain(stream, state) {
if (state.length === 0 && state.needDrain) {
state.needDrain = false;
stream.emit('drain');
}
}
// if there's something in the buffer waiting, then process it
function clearBuffer(stream, state) {
state.bufferProcessing = true;
if (stream._writev && state.buffer.length > 1) {
// Fast case, write everything using _writev()
var cbs = [];
for (var c = 0; c < state.buffer.length; c++)
cbs.push(state.buffer[c].callback);
// count the one we are adding, as well.
// TODO(isaacs) clean this up
state.pendingcb++;
doWrite(stream, state, true, state.length, state.buffer, '', function(err) {
for (var i = 0; i < cbs.length; i++) {
state.pendingcb--;
cbs[i](err);
}
});
// Clear buffer
state.buffer = [];
} else {
// Slow case, write chunks one-by-one
for (var c = 0; c < state.buffer.length; c++) {
var entry = state.buffer[c];
var chunk = entry.chunk;
var encoding = entry.encoding;
var cb = entry.callback;
var len = state.objectMode ? 1 : chunk.length;
doWrite(stream, state, false, len, chunk, encoding, cb);
// if we didn't call the onwrite immediately, then
// it means that we need to wait until it does.
// also, that means that the chunk and cb are currently
// being processed, so move the buffer counter past them.
if (state.writing) {
c++;
break;
}
}
if (c < state.buffer.length)
state.buffer = state.buffer.slice(c);
else
state.buffer.length = 0;
}
state.bufferProcessing = false;
}
Writable.prototype._write = function(chunk, encoding, cb) {
cb(new Error('not implemented'));
};
Writable.prototype._writev = null;
Writable.prototype.end = function(chunk, encoding, cb) {
var state = this._writableState;
if (util.isFunction(chunk)) {
cb = chunk;
chunk = null;
encoding = null;
} else if (util.isFunction(encoding)) {
cb = encoding;
encoding = null;
}
if (!util.isNullOrUndefined(chunk))
this.write(chunk, encoding);
// .end() fully uncorks
if (state.corked) {
state.corked = 1;
this.uncork();
}
// ignore unnecessary end() calls.
if (!state.ending && !state.finished)
endWritable(this, state, cb);
};
function needFinish(stream, state) {
return (state.ending &&
state.length === 0 &&
!state.finished &&
!state.writing);
}
function prefinish(stream, state) {
if (!state.prefinished) {
state.prefinished = true;
stream.emit('prefinish');
}
}
function finishMaybe(stream, state) {
var need = needFinish(stream, state);
if (need) {
if (state.pendingcb === 0) {
prefinish(stream, state);
state.finished = true;
stream.emit('finish');
} else
prefinish(stream, state);
}
return need;
}
function endWritable(stream, state, cb) {
state.ending = true;
finishMaybe(stream, state);
if (cb) {
if (state.finished)
process.nextTick(cb);
else
stream.once('finish', cb);
}
state.ended = true;
}

Some files were not shown because too many files have changed in this diff Show More