apache/spark — pull request #1115 by gao8658 (2014-06-18 16:22:47+00:00)
Title: Branch 1.0 Add ZLIBCompressionCodec code

Hi ,ALL,I try to add Spark ZLIBCompressionCodec
the code is here and the jzlib lib is from http://www.jcraft.com/jzlib/
The code can run on Spark source code ,and the test can pass.

code :

@DeveloperApi
class ZLIBCompressionCodec(conf: SparkConf) extends CompressionCodec {

import java.io._
import com.jcraft.jzlib._
@DeveloperApi
class ZLIBCompressionCodec(conf: SparkConf) extends CompressionCodec {

override def compressedOutputStream(s: OutputStream): OutputStream = {
val zOut = new ZOutputStream(s, JZlib.Z_BEST_COMPRESSION)
zOut
}
override def compressedInputStream(s: InputStream): InputStream = new ZInputStream(s)
}

testsuite :

class CompressionCodecSuite extends FunSuite {
val conf = new SparkConf(false)

def testCodec(codec: CompressionCodec) {
// Write 1000 integers to the output stream, compressed.
val outputStream = new ByteArrayOutputStream()
val out = codec.compressedOutputStream(outputStream)
for (i <- 1 until 1000) {
out.write(i % 256)
}
out.close()

// Read the 1000 integers back.
val inputStream = new ByteArrayInputStream(outputStream.toByteArray)
val in = codec.compressedInputStream(inputStream)
for (i <- 1 until 1000) {
assert(in.read() === i % 256)
}
in.close()
}

test("default compression codec") {
val codec = CompressionCodec.createCodec(conf)
assert(codec.getClass === classOf[LZFCompressionCodec])
testCodec(codec)
}

test("lzf compression codec") {
val codec = CompressionCodec.createCodec(conf, classOf[LZFCompressionCodec].getName)
assert(codec.getClass === classOf[LZFCompressionCodec])
testCodec(codec)
}

test("snappy compression codec") {
val codec = CompressionCodec.createCodec(conf, classOf[SnappyCompressionCodec].getName)
assert(codec.getClass === classOf[SnappyCompressionCodec])
testCodec(codec)
}
test("zlib compression codec") {
val codec = CompressionCodec.createCodec(conf, classOf[ZLIBCompressionCodec].getName)
assert(codec.getClass === classOf[ZLIBCompressionCodec])
testCodec(codec)
}
}