src/main/java/com/ociweb/jfast/stream/FASTReaderReactor.java
public final class FASTReaderReactor {

    /**
     * Read up to the end of the next sequence or message (eg. a repeating
     * group)
     * 
     * Rules for making client compatible changes to templates. - Field can be
     * demoted to more general common value before the group. - Field can be
     * promoted to more specific value inside sequence - Field order inside
     * group can change but can not cross sequence boundary. - Group boundaries
     * can be added or removed.
     * 
     * Note nested sequence will stop once for each of the sequences therefore
     * at the bottom hasMore may not have any new data but is only done as a
     * notification that the loop has completed.
     * 
     * @return
     */
    
    private final FASTDecoder decoder; 
    public final PrimitiveReader reader;//the reader is non-blocking but awkward to use directly.
        
    public FASTReaderReactor(FASTDecoder decoder, PrimitiveReader reader) {
        this.decoder=decoder;
        this.reader=reader;
        
    }
    
    //TODO: B, support zero copy mapping by (reader adds gaps to rb, writer can skip inputs from rb) add to config
    
    public AtomicBoolean start(final ThreadPoolExecutor executorService, final PrimitiveReader reader) {
        
        final AtomicBoolean isAlive = new AtomicBoolean(true);
        
        reader.setInputPolicy(new InputBlockagePolicy() {
            Object lock = new Object();

            @Override
            public void detectedInputBlockage(int need, FASTInput input) {
                //TODO: C, create these extra threads on startup and pause them until this moment, this prevents creation and gc at runtime
                //TODO: C, formalize this pattern in a new M:N ThreadPoolExecutor, send in the number of cores you wish to target for parsing not threads.
                //TODO: C, once this threading is in place can the move next also be added to the same pool if we desire? This may give us locality across both calls
                //System.err.println("Begin block");
                synchronized(lock) {
                    executorService.setMaximumPoolSize(executorService.getMaximumPoolSize()+1);
                }                
            }
             
            @Override
            public void resolvedInputBlockage(FASTInput input) {
                System.err.println("Release block");
                synchronized(lock) {
                    executorService.setMaximumPoolSize(executorService.getMaximumPoolSize()-1);
                }
            }
            
        });
        
        final Runnable run = buildRunnable(executorService, isAlive);        
        executorService.execute(run);        
        return isAlive;
    }

    private Runnable buildRunnable(final ThreadPoolExecutor executorService, final AtomicBoolean isAlive) {
        final Runnable run = new Runnable() {

            final FASTDecoder decoder2 = FASTReaderReactor.this.decoder;
            final PrimitiveReader reader2 = FASTReaderReactor.this.reader;

            @Override
            public void run() {

                    int f=0;
                    
                    int c = 0x1FFFFF;
                    while (--c>=0)  { 
                        
                        if ((f=decoder2.decode(reader2))<=0) { //break on eof or no room to read
                           
                            break;
                        }  
                    }
                       
                    if (f>=0) {
                        executorService.execute(this);//buildRunnable(executorService,isAlive));
                    } else {
                        isAlive.set(false);
                    }

            }
            
        };
        return run;
    }

    public static int pump(FASTReaderReactor reactor) {
            return reactor.decoder.decode(reactor.reader);
    }
    
    public Pipe[] ringBuffers() {
    	return PipeBundle.buffers(decoder.ringBuffers);
    }
    

}
src/main/java/com/ociweb/jfast/primitive/adapter/FASTInputStream.java
public class FASTInputStream implements FASTInput {

	private InputStream inst;
	private byte[] targetBuffer;
	private boolean eof = false;
	private long total;
	
	public FASTInputStream(InputStream inst) {
		this.inst = inst;
	}
	
	public void replaceStream(InputStream inst) {
		this.inst = inst;
	}
	
	public int fill(int offset, int len) {
		try {
			
		    int avail = inst.available();
		    if (avail>0) {
		        //   len = len<avail?len:avail;
		        len = len + ((avail-len) & (avail-len)>>31);//branchless
		    }
					    
			//Only fill with the bytes avail.			
			int result = inst.read(targetBuffer, offset, len);
			if (result<0) {
				eof = true;;
				return 0;
			}
			total+=result;
			return result;
		} catch (IOException e) {
			throw new FASTException(e);
		}
	}

	@Override
	public void init(byte[] targetBuffer) {
		this.targetBuffer = targetBuffer;
	}

	@Override
	public boolean isEOF() {
		return eof;
	}
	
	public long totalBytes() {
	    return total;
	}

    @Override
    public int blockingFill(int offset, int count) {
        int result;
        try {
            result = inst.read(targetBuffer, offset, count);
        } catch (IOException e) {
            throw new FASTException(e);
        }
        if (result<0) {
            eof = true;;
            return 0;
        }
        total+=result;
        return result;
    }
	
}
src/main/java/com/ociweb/jfast/error/FASTException.java
public class FASTException extends RuntimeException {

	private static final long serialVersionUID = 8294717647055016321L;

	public FASTException() {
		
	}

	public FASTException(Throwable e) {
		super(e);
	}

	public FASTException(FASTError error) {
		super(error.toString());
	}

	public FASTException(String error) {
		super(error);
	}
}
src/main/java/com/ociweb/jfast/generator/FASTClassLoader.java
    public class FASTClassLoader extends ClassLoader{

    	private static final Logger log = LoggerFactory.getLogger(FASTClassLoader.class);
    	
        public static final String GENERATED_PACKAGE = "com.ociweb.jfast.generator";
        public static final String SIMPLE_READER_NAME = "FASTReaderGeneratedDispatch";
        public static final String SIMPLE_WRITER_NAME = "FASTWriterGeneratedDispatch";
        
        public static final String READER = GENERATED_PACKAGE+'.'+SIMPLE_READER_NAME;
        public static final String WRITER = GENERATED_PACKAGE+'.'+SIMPLE_WRITER_NAME;
                
        final byte[] catBytes;// serialized catalog for the desired templates XML
        final boolean forceCompile;
        final boolean exportSource;
        public static boolean fullCompiled = false;
        
        static final File workingFolder = new File(new File(System.getProperty("java.io.tmpdir")),"jFAST");
        static {
            workingFolder.mkdirs();
        }
        
        public FASTClassLoader(byte[] catBytes, ClassLoader parent) {
            this(catBytes,parent,false);
        }
        
        public FASTClassLoader(byte[] catBytes, ClassLoader parent, boolean forceCompile) {
            super(parent);
            this.catBytes = catBytes;
            this.exportSource = Boolean.getBoolean("FAST.exportSource");
            this.forceCompile = forceCompile | exportSource | Boolean.getBoolean("FAST.forceCompile");
            log.trace("Created new FASTClassLoader forceCompile:"+forceCompile+" exportSource:"+exportSource);
            
        }        
        
        @SuppressWarnings("boxing")
        @Override
        public Class loadClass(String name) throws ClassNotFoundException {
            //if we want a normal class use the normal class loader
            if(!name.contains("GeneratedDispatch") || !name.contains("jfast.generator.FAST")) {
                return super.loadClass(name);
            }
            
            //if class is found and matches use it.
            String simpleClassName = name.substring(name.lastIndexOf('.')+1);
            
            
            File classFile = targetFile(simpleClassName, "class");
            
            //CAUTION: only use force compile when you need deep testing it can be very slow.
            if ((fullCompiled || !forceCompile) && classFile.exists()) {
                log.trace("Reading class from: {}",classFile);
                
                byte[] classData = readClassBytes(classFile);
                                    
                //returning with defineClass helps reduce the risk that we may try to define the name again.
                return defineClass(name, classData , 0, classData.length);                    
            }
                        
            //if we have a compiler then regenerate the source and class based on the templates.
            JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
            if (null!=compiler && compiler.getSourceVersions().contains(SourceVersion.RELEASE_6)) {
                log.trace("Compile class to: {}",classFile);
                
                List<String> optionList = new ArrayList<String>();
                optionList.addAll(Arrays.asList("-classpath", System.getProperty("java.class.path"),
                                                "-d", workingFolder.toString(),
                                                "-target","1.6",
                                                "-source","1.6"
                                                ));                

                List<JavaFileObject> toCompile = new ArrayList<JavaFileObject>();

                {//scoped to help GC 
                	FASTReaderDispatchGenerator readGenerator = new FASTReaderDispatchGenerator(catBytes, toCompile, TemplateCatalogConfig.buildRingBuffers(new TemplateCatalogConfig(catBytes), (byte)8, (byte)18));
                	SimpleSourceFileObject sourceReaderFileObject = new SimpleSourceFileObject(FASTClassLoader.SIMPLE_READER_NAME,
                													   						   readGenerator.generateFullSource(new StringBuilder()));
                	toCompile.add(sourceReaderFileObject);
                	
                }
                
                {//scoped to help GC
                	FASTWriterDispatchGenerator writeGenerator = new FASTWriterDispatchGenerator(catBytes, new TemplateCatalogConfig(catBytes), toCompile);
                	SimpleSourceFileObject sourceWriterFileObject = new SimpleSourceFileObject(FASTClassLoader.SIMPLE_WRITER_NAME,
									             										       writeGenerator.generateFullSource(new StringBuilder()));
					toCompile.add(sourceWriterFileObject);
                }
                log.debug("Begin full compile of {} files",toCompile.size());
                
                if (exportSource) {
                	for(JavaFileObject jfo:toCompile) {
                        try {
                            exportSourceToClassFolder(jfo.getName(),jfo.getCharContent(false).toString());
                        } catch (IOException e) {
                            log.trace("Unable to write source {}",e.getMessage());
                        }	                		
                	}
                }
                DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<JavaFileObject>();
                
                
                if (compiler.getTask(null, null, diagnostics, optionList, null, toCompile).call()) {
                    byte[] classData = readClassBytes(classFile);
                    Class result =  defineClass(name, classData , 0, classData.length);
                    fullCompiled = true; //only set after success
                    log.debug("Finished full compile");
                    return result;
                } else {
                    reportCompileError(diagnostics.getDiagnostics());      
                }
            }
            throw new ClassNotFoundException();
        }

        private void reportCompileError(List<Diagnostic<? extends JavaFileObject>> diagnosticList)
                throws ClassNotFoundException {
        	
            Iterator<Diagnostic<? extends JavaFileObject>> iter = diagnosticList.iterator();
            while (iter.hasNext()) {
                System.err.println(iter.next());
            }
            //did not compile due to error
            if (!diagnosticList.isEmpty()) {
                throw new ClassNotFoundException(diagnosticList.get(0).toString());
            } else {
                throw new ClassNotFoundException("Compiler error");
            }
        }

        private byte[] readClassBytes(File classFile) throws ClassNotFoundException {
            byte[] classData = new byte[(int)classFile.length()];
            try {
                FileInputStream input = new FileInputStream(classFile);
                input.read(classData);
                input.close(); 
            } catch (Exception e) {
                throw new ClassNotFoundException("Unable to read class file.", e);
            }
            return classData;
        }


//        if (exportSource) {
//            for(JavaFileObject jfo:toCompile) {
//                try {
//                    exportSourceToClassFolder(jfo.getName(),jfo.getCharContent(false).toString());
//                } catch (IOException e) {
//                    log.trace("Unable to write source {}",e.getMessage());
//                }                           
//            }
//        }
        
        private void exportSourceToClassFolder(String name, String content) {
            try {
                File sourceFile = targetFile(name, "java");
               // System.err.println("Wrote source to: "+sourceFile);
                FileWriter out = new FileWriter(sourceFile);
                out.write(content);
                out.close();
            } catch (IOException e1) {
            	log.trace("uanble to export source: {} ",e1.getMessage());
            }
        }
        
        private static File targetFile(String name, String ext) {
            return new File(workingFolder,GENERATED_PACKAGE.replace('.', File.separatorChar)+File.separatorChar+name+"."+ext);
        }
        
        public static void logCompileError(List<Diagnostic<? extends JavaFileObject>> diagnostics) {
		    Iterator<Diagnostic<? extends JavaFileObject>> iter = diagnostics.iterator();
		    while (iter.hasNext()) {
		        System.err.println(iter.next());
		    }
		}

        public static void deleteFiles() {
            targetFile(SIMPLE_READER_NAME,"class").delete();
            targetFile(SIMPLE_READER_NAME,"java").delete();
            
            targetFile(SIMPLE_WRITER_NAME,"class").delete();
            targetFile(SIMPLE_WRITER_NAME,"java").delete(); 
        }
        

    }
src/main/java/com/ociweb/jfast/catalog/loader/TemplateLoader.java
public class TemplateLoader {

	private static final Logger log = LoggerFactory.getLogger(TemplateLoader.class);
	
    private static final int BUILDING_EXCEPTION = -6;
    private static final int FILE_NOT_FOUND = -5;
    private static final int FILE_REQUIRED_NOT_DIRECTORY = -4;
    private static final int NO_WRITE_RIGHTS = -3;
    private static final int MISSING_REQ_ARG = -2;
    private static final int MISSING_ARG_VALUE = -1;

    /**
     * Load templates into catalog file.
     * 
     * The jFAST engine will support dynamic template changes among those in the
     * catalog. Building the catalog also validates no conflicts exist between
     * the contained templates. The catalog file can be replaced while jFAST is
     * running however it will not be accessed unless an unknown template id is
     * encountered or it is explicitly requested.
     * 
     * -s -source <folder or XML file as exclusive input into new catalog> -c
     * -catalog <absolute path to catalog file>
     * 
     * @param args
     */

    public static void main(String[] args) {

        File catalog = new File(getReqArg("-catalog", "-c", args));
        String source = getReqArg("-source", "-s", args);

        if (!catalog.canWrite()) {
            printHelp("Unable to write to location: " + catalog);
            System.exit(NO_WRITE_RIGHTS);
        }
        if (catalog.isDirectory()) {
            printHelp("Catalog must be a file not a directory: " + catalog);
            System.exit(FILE_REQUIRED_NOT_DIRECTORY);
        }

        ClientConfig clientConfig = new ClientConfig(); //TODO: B, load from file or args?
        
        try {
            buildCatalog(new FileOutputStream(catalog), source, clientConfig);
        } catch (Exception e) {
            e.printStackTrace();
            System.exit(BUILDING_EXCEPTION);
        }
    }

    public static FieldReferenceOffsetManager buildCatalog(OutputStream outputStream, String source, ClientConfig clientConfig) throws ParserConfigurationException,
            SAXException, IOException {
        log.trace("source file:{}",source);               
        InputStream sourceInputStream = TemplateLoader.class.getResourceAsStream(source);     
        
        File folder = null;
        if (null==sourceInputStream) {
            folder = new File(source);
            if (folder.exists() && !folder.isDirectory()) {
                sourceInputStream = new FileInputStream(source);
            }        
        }
        
        GZIPOutputStream gZipOutputStream = new GZIPOutputStream(outputStream);
		FASTOutput output = new FASTOutputStream(gZipOutputStream);
		TemplateHandler handler = new TemplateHandler();   
		
		
		SAXParserFactory spfac = SAXParserFactory.newInstance();
		SAXParser sp = spfac.newSAXParser();
		if (null!= sourceInputStream) {
		    sp.parse(sourceInputStream, handler);   
		} else {
		    for (File f : folder.listFiles()) {
		        if (f.isFile()) {
		            sp.parse(f, handler);
		        }
		    }
		}
		
		PrimitiveWriter writer = new PrimitiveWriter(4096, output, false);
		TemplateCatalogConfig.writeTemplateCatalog(handler, clientConfig.getBytesGap(), clientConfig.getBytesLength(), writer, clientConfig);
		gZipOutputStream.close();
		
		return TemplateHandler.from(handler, clientConfig.getPreableBytes());
    }

	public static void buildCatalog(OutputStream outputStream,
									InputStream inputStream, ClientConfig clientConfig)
									throws ParserConfigurationException, SAXException, IOException {

		GZIPOutputStream gZipOutputStream = new GZIPOutputStream(outputStream);
		FASTOutput output = new FASTOutputStream(gZipOutputStream);
		TemplateHandler handler = new TemplateHandler();

		SAXParserFactory spfac = SAXParserFactory.newInstance();
		SAXParser sp = spfac.newSAXParser();

		sp.parse(inputStream, handler);

		PrimitiveWriter writer = new PrimitiveWriter(4096, output, false);
		TemplateCatalogConfig.writeTemplateCatalog(handler, clientConfig.getBytesGap(), clientConfig.getBytesLength(), writer, clientConfig);
		gZipOutputStream.close();
	}
    

    private static void printHelp(String message) {
        System.out.println(message);
        System.out.println();
        System.out.println("Usage:");
        System.out.println("       TemplateLoader -s <templates folder or template file> -c <catalog file>");
        System.out.println();
        System.out.println("Arguments:");
        System.out.println("          -s or -source       full path to xml template or folder of templates.");
        System.out.println("          -c or -catalog      full path to catalog file to write over.");
        System.out.println();
    }

    private static String getReqArg(String longName, String shortName, String[] args) {
        String prev = null;
        for (String token : args) {
            if (longName.equals(prev) || shortName.equals(prev)) {
                if (token == null || token.trim().length() == 0 || token.startsWith("-")) {
                    printHelp("Expected value not found");
                    System.exit(MISSING_ARG_VALUE);
                }
                return token.trim();
            }
            prev = token;
        }
        printHelp("Expected value not found");
        System.exit(MISSING_REQ_ARG);
        return null;
    }

	public static byte[] buildCatBytes(String source, ClientConfig clientConfig) {
		ByteArrayOutputStream catalogBuffer = new ByteArrayOutputStream(4096);
	    try {
	        buildCatalog(catalogBuffer, source, clientConfig);
	    } catch (Exception e) {
	        e.printStackTrace();
	    }
	    assert(catalogBuffer.size() > 0);
	    return catalogBuffer.toByteArray();
	}
	
	public static byte[] buildCatBytes(InputStream input, ClientConfig clientConfig) {
		ByteArrayOutputStream catalogBuffer = new ByteArrayOutputStream(4096);
	    try {
	        buildCatalog(catalogBuffer, input, clientConfig);
	    } catch (Exception e) {
	        e.printStackTrace();
	    }
	    assert(catalogBuffer.size() > 0);
	    return catalogBuffer.toByteArray();
	}
}
src/main/java/com/ociweb/jfast/generator/DispatchLoader.java
public class DispatchLoader {

    private static final boolean FORCE_COMPILE = true;
    private static final Logger log = LoggerFactory.getLogger(DispatchLoader.class);

    public static FASTDecoder loadDispatchReader(byte[] catalog, PipeBundle ringBuffers) {
        //always try to load the generated reader because it will be faster 
        try {
            return loadGeneratedReaderDispatch(catalog, FASTClassLoader.READER, ringBuffers);
        } catch (Exception e) {
        	log.error("Attempted to load dispatch reader.", e);
            return new FASTReaderInterpreterDispatch(catalog, ringBuffers);
        }
    }
    
    public static FASTDecoder loadDispatchReaderDebug(byte[] catalog, PipeBundle ringBuffers) {
        ///NOTE: when there is only 1 template the compiled dispatch may continue to work if the templateId is wrong
        //       because it does not do extra checking.  The interpreted one will use the templateId as defined.
        return new FASTReaderInterpreterDispatch(catalog, ringBuffers);
    }

    public static FASTEncoder loadDispatchWriter(byte[] catalog) {
        //always try to load the generated reader because it will be faster 
        try {
            return loadGeneratedWriterDispatch(catalog, FASTClassLoader.WRITER);
        } catch (Exception e) {
        	log.error("Attempted to load dispatch reader.", e);
            return loadDispatchWriterDebug(catalog);
        }
    }
    
    public static FASTEncoder loadDispatchWriterDebug(byte[] catBytes) {
		return new FASTWriterInterpreterDispatch(new TemplateCatalogConfig(catBytes));
    }
    
    public static <T> T loadGeneratedReaderDispatch(byte[] catBytes, String type, PipeBundle ringBuffers)
            throws ReflectiveOperationException, SecurityException {
        
        ClassLoader parentClassLoader = FASTDecoder.class.getClassLoader();
        
        try {
            Class generatedClass = new FASTClassLoader(catBytes, parentClassLoader).loadClass(type);
            
            int[] catHash = (int[])generatedClass.getField("hashedCat").get(null);
            int[] expectedHash = GeneratorData.hashCatBytes(catBytes);
            
            if (!Arrays.equals(catHash, expectedHash)) {
            	log.trace("Catalog mistmatch, attempting source regeneration and recompile.");
                //the templates catalog this was generated for does not match the current value so force a recompile
                generatedClass = new FASTClassLoader(catBytes, parentClassLoader, FORCE_COMPILE).loadClass(type);
            }
                       
            
            return (T)generatedClass.getConstructor(catBytes.getClass(), ringBuffers.getClass()).newInstance(catBytes, ringBuffers);
        } catch (Throwable t) {
        	log.error("Error in creating instance, attempting source regeneration and recompile.", t);
            //can not create instance because the class is no longer compatible with the rest of the code base so force a recompile
            Class generatedClass = new FASTClassLoader(catBytes, parentClassLoader, FORCE_COMPILE).loadClass(type);
            return (T)generatedClass.getConstructor(catBytes.getClass(), ringBuffers.getClass()).newInstance(catBytes, ringBuffers);
        }
    }
    
    public static <T> T loadGeneratedWriterDispatch(byte[] catBytes, String type)
            throws ReflectiveOperationException, SecurityException {
        
        ClassLoader parentClassLoader = FASTEncoder.class.getClassLoader();
        
        try {
            Class generatedClass = new FASTClassLoader(catBytes, parentClassLoader).loadClass(type);
            
            int[] catHash = (int[])generatedClass.getField("hashedCat").get(null);
            int[] expectedHash = GeneratorData.hashCatBytes(catBytes);
            
            if (!Arrays.equals(catHash, expectedHash)) {
            	log.error("Catalog mistmatch, attempting source regeneration and recompile.");
                //the templates catalog this was generated for does not match the current value so force a recompile
                generatedClass = new FASTClassLoader(catBytes, parentClassLoader, FORCE_COMPILE).loadClass(type);
            }
                       
            
            return (T)generatedClass.getConstructor(catBytes.getClass()).newInstance(catBytes);
        } catch (Throwable t) {
        	log.error("Error in creating instance, attempting source regeneration and recompile.", t);
            //can not create instance because the class is no longer compatible with the rest of the code base so force a recompile
            Class generatedClass = new FASTClassLoader(catBytes, parentClassLoader, FORCE_COMPILE).loadClass(type);
            return (T)generatedClass.getConstructor(catBytes.getClass()).newInstance(catBytes);
        }
    }


}
src/main/java/com/ociweb/jfast/catalog/loader/ClientConfig.java
public class ClientConfig {

    private short preambleBytes;
    
    private int bytesLengthMax = 4096;
    private int bytesGap = 64;
    
    private static final int NONE = -1;
    private int catalogId = NONE;

    public ClientConfig() {
    }
    
    public ClientConfig(int bytesLength, int bytesGap) {
        this.bytesLengthMax = bytesLength;
        this.bytesGap = bytesGap;
        
    }
    
    public ClientConfig(PrimitiveReader reader) {
        
        preambleBytes = (short)PrimitiveReader.readIntegerUnsigned(reader);
        
        bytesLengthMax = PrimitiveReader.readIntegerUnsigned(reader);
        bytesGap = PrimitiveReader.readIntegerUnsigned(reader);
        
        catalogId = PrimitiveReader.readIntegerSigned(reader);

    }

    public void save(PrimitiveWriter writer) {
        
        PrimitiveWriter.writeIntegerUnsigned(preambleBytes, writer);

        PrimitiveWriter.writeIntegerUnsigned(bytesLengthMax, writer);
        PrimitiveWriter.writeIntegerUnsigned(bytesGap, writer);
        
        PrimitiveWriter.writeIntegerSigned(catalogId, writer);
    
        
    }
    
    public short getPreableBytes() {
        return preambleBytes;
    }

    public void setPreableBytes(short preableBytes) {
        this.preambleBytes = preableBytes;
    }
    
    public int getBytesLength() {
        return this.bytesLengthMax;
    }
    
    public int getBytesGap() {
        return this.bytesGap;
    }

    public void setCatalogTemplateId(int id) {
        if (NONE == id) {
            throw new FASTException("Catalog template Id may not be: "+NONE);
        }
        catalogId = id;
    }

}
src/main/java/com/ociweb/jfast/catalog/loader/TemplateCatalogConfig.java
public class TemplateCatalogConfig {

    ///////////
    //properties that should only be set after reading the documentation.
    //Change impact generated code so the changes must be done only once before the catBytes are generated.
    ///////////
    
    public final ClientConfig clientConfig;
    
    
    private final DictionaryFactory dictionaryFactory;
    private final int maxTemplatePMapSize;
    private final int maxNonTemplatePMapSize;
    private final int maxPMapDepth;
    private final long maxFieldId; 
 
    private final LongHashTable templateToStartIdx;
    
    public final int[] templateScriptEntries;
    public final int[] templateScriptEntryLimits;

    final int[] scriptTokens;
    final long[] scriptFieldIds;
    private final String[] scriptFieldNames;
    private final String[] scriptDictionaryNames;
    private final int templatesInCatalog;

    
    private final int[][] dictionaryMembers;

    private final FieldReferenceOffsetManager from;
    
    public TemplateCatalogConfig(byte[] catBytes) {
        
        FASTInputStream inputStream;
        try {
            inputStream = new FASTInputStream(new GZIPInputStream(new ByteArrayInputStream(catBytes),catBytes.length));
        } catch (IOException e) {
            throw new FASTException(e);
        }
        
        
        PrimitiveReader reader = new PrimitiveReader(1024, inputStream, 0);

        //given an index in the script lookup the tokens, fieldIds or fieldNames
        int fullScriptLength = PrimitiveReader.readIntegerUnsigned(reader);
                        
        scriptTokens = new int[fullScriptLength];
        scriptFieldIds = new long[fullScriptLength];
        scriptFieldNames = new String[fullScriptLength];
        scriptDictionaryNames = new String[fullScriptLength];
        
        //given the template id from the template file look up the 
        //script starts and limits
        templatesInCatalog = PrimitiveReader.readIntegerUnsigned(reader);
        
        //how many bits would be needed to store this many templateIds
        int bitsForTemplates = 32 - Integer.numberOfLeadingZeros(templatesInCatalog - 1);
        
        templateToStartIdx = new LongHashTable(bitsForTemplates+2); //add 2 so we have plenty of extra room for hashes
        
        
        templateScriptEntries = new int[templatesInCatalog];
        templateScriptEntryLimits = new int[templatesInCatalog];
        
        loadTemplateScripts(reader);

        int dictionaryCount = PrimitiveReader.readIntegerUnsigned(reader);
        dictionaryMembers = new int[dictionaryCount][];

        loadDictionaryMembers(reader);

        maxFieldId = PrimitiveReader.readLongUnsigned(reader);
        // it is assumed that template PMaps are smaller or larger than the
        // other PMaps so these are kept separate
        maxTemplatePMapSize = PrimitiveReader.readIntegerUnsigned(reader);
        maxNonTemplatePMapSize = PrimitiveReader.readIntegerUnsigned(reader);
        maxPMapDepth = PrimitiveReader.readIntegerUnsigned(reader);
		DictionaryFactory df = new DictionaryFactory();
		TemplateCatalogConfig.load(df,reader);

        dictionaryFactory = df;
                
        clientConfig = new ClientConfig(reader);
        
        //must be done after the client config construction
        from = TemplateCatalogConfig.createFieldReferenceOffsetManager(this);
        
    }
    
    @Deprecated //for testing only
    public TemplateCatalogConfig(DictionaryFactory dcr, int nonTemplatePMapSize, int[][] dictionaryMembers,
                                int[] fullScript, int maxNestedGroupDepth,
                                int maxTemplatePMapSize, int templatesCount, ClientConfig clientConfig) {
        
        this.scriptTokens = fullScript;
        this.maxNonTemplatePMapSize  = nonTemplatePMapSize;
        this.dictionaryMembers = dictionaryMembers;
        this.maxPMapDepth = maxNestedGroupDepth;
        this.templatesInCatalog=templatesCount;
        this.templateToStartIdx=null;
        this.scriptFieldNames=null;
        this.scriptDictionaryNames=null;
        this.templateScriptEntries=null;
        this.templateScriptEntryLimits=null;
        this.scriptFieldIds=null;
        this.maxTemplatePMapSize = maxTemplatePMapSize;
        this.maxFieldId=-1;
        this.dictionaryFactory = dcr;
        this.clientConfig = clientConfig;
        
        this.from = TemplateCatalogConfig.createFieldReferenceOffsetManager(this);

    }

	public byte[] ringByteConstants() {
		return DictionaryFactory.initConstantByteArray(dictionaryFactory);
	}

    // Assumes that the tokens are already loaded and ready for use.
    private void loadTemplateScripts(PrimitiveReader reader) {

        int i = templatesInCatalog;
        while (--i >= 0) {
            // look up for script index given the templateId
            long templateId = PrimitiveReader.readLongUnsigned(reader);
            LongHashTable.setItem(templateToStartIdx, templateId, templateScriptEntries[i] = PrimitiveReader.readIntegerUnsigned(reader));
            templateScriptEntryLimits[i] = PrimitiveReader.readIntegerUnsigned(reader);
        }
        
        //Must be ordered in order to be useful 
        Arrays.sort(templateScriptEntries);
        Arrays.sort(templateScriptEntryLimits);

        StringBuilder builder = new StringBuilder();//TODO: B, this is now producing garbage! Temp space must be held by temp space owner!
        i = getScriptTokens().length;
        while (--i >= 0) {
            getScriptTokens()[i] = PrimitiveReader.readIntegerSigned(reader);
            scriptFieldIds[i] = PrimitiveReader.readIntegerUnsigned(reader);
            scriptFieldNames[i] = readUTF8(reader, builder);;
            scriptDictionaryNames[i] = readUTF8(reader, builder);;
            
        }

        // System.err.println("script tokens/fields "+scriptTokens.length);//46
        // System.err.println("templateId idx start/stop count "+this.templateStartIdx.length);//128

    }

	public String readUTF8(PrimitiveReader reader, StringBuilder builder) {
		int len = PrimitiveReader.readIntegerUnsigned(reader);
		String name ="";
		if (len>0) {
		    builder.setLength(0);
		    {
		        byte[] tmp = new byte[len];                    
		        PrimitiveReader.readByteData(tmp,0,len,reader); //read bytes into array
		        
		        long charAndPos = 0;  //convert bytes to chars
		        while (charAndPos>>32 < len  ) { 
		            charAndPos = Pipe.decodeUTF8Fast(tmp, charAndPos, Integer.MAX_VALUE);
		            builder.append((char)charAndPos);

		        }
		    }
		    name = builder.toString();
		}
		return name;
	}

    // // stream message* | block*
    // // block BlockSize message+
    // // message segment
    // // segment PresenceMap TemplateIdentifier? (field | segment)*
    // //* field integer | string | delta | ScaledNumber | ByteVector
    // //* integer UnsignedInteger | SignedInteger
    // //* string ASCIIString | UnicodeString
    // //* delta IntegerDelta | ScaledNumberDelta | ASCIIStringDelta
    // |ByteVectorDelta

    public static void save(PrimitiveWriter writer, long biggestId, int uniqueTemplateIds, long biggestTemplateId,
            DictionaryFactory df, int maxTemplatePMap, int maxNonTemplatePMap, int[][] tokenIdxMembers,
            int[] tokenIdxMemberHeads, int[] catalogScriptTokens, long[] catalogScriptFieldIds, String[] catalogScriptFieldNames, String[] dictionaryNames,
            int scriptLength,  LongHashTable templateToOffset, LongHashTable templateToLimit , int maxPMapDepth, ClientConfig clientConfig) {    
        
        saveTemplateScripts(writer, uniqueTemplateIds, biggestTemplateId, catalogScriptTokens, 
                catalogScriptFieldIds, catalogScriptFieldNames, dictionaryNames,
                scriptLength, templateToOffset, templateToLimit);

        saveDictionaryMembers(writer, tokenIdxMembers, tokenIdxMemberHeads);

        PrimitiveWriter.writeLongUnsigned(biggestId, writer);
        // System.err.println("save pmap sizes "+maxTemplatePMap+" "+maxNonTemplatePMap);
        PrimitiveWriter.writeIntegerUnsigned(maxTemplatePMap, writer);
        PrimitiveWriter.writeIntegerUnsigned(maxNonTemplatePMap, writer);
        PrimitiveWriter.writeIntegerUnsigned(maxPMapDepth, writer);

        TemplateCatalogConfig.save(df, writer);        
        clientConfig.save(writer);

    }

    private static void saveProperties(PrimitiveWriter writer, Properties properties) {
                
        Set<String> keys = properties.stringPropertyNames();
        PrimitiveWriter.writeIntegerUnsigned(keys.size(), writer);
        for(String key: keys) {
            PrimitiveWriter.writeIntegerUnsigned(key.length(), writer);
            PrimitiveWriter.ensureSpace(key.length(),writer);
            
            {
                //convert from chars to bytes
                //writeByteArrayData()
                int len = key.length();
                int limit = writer.limit;
                int c = 0;
                while (c < len) {
                    limit = Pipe.encodeSingleChar((int) key.charAt(c++), writer.buffer, 0xFFFFFFFF, limit);
                }
                writer.limit = limit;
            }
            
            String prop = properties.getProperty(key);
            PrimitiveWriter.writeIntegerUnsigned(prop.length(), writer);
            PrimitiveWriter.ensureSpace(prop.length(),writer);
            
            {
                //convert from chars to bytes
                //writeByteArrayData()
                int len = prop.length();
                int limit = writer.limit;
                int c = 0;
                while (c < len) {
                    limit = Pipe.encodeSingleChar((int) prop.charAt(c++), writer.buffer, 0xFFFFFFFF, limit);
                }
                writer.limit = limit;
            }
            
        }
        
    }

    private static void saveDictionaryMembers(PrimitiveWriter writer, int[][] tokenIdxMembers, int[] tokenIdxMemberHeads) {
        // save count of dictionaries
        int dictionaryCount = tokenIdxMembers.length;
        PrimitiveWriter.writeIntegerUnsigned(dictionaryCount, writer);
        //
        int d = dictionaryCount;
        while (--d >= 0) {
            int[] members = tokenIdxMembers[d];
            int h = tokenIdxMemberHeads[d];
            PrimitiveWriter.writeIntegerUnsigned(h, writer);// length of reset script (eg member list)
            while (--h >= 0) {
                PrimitiveWriter.writeIntegerSigned(members[h], writer);
            }
        }
    }

    private void loadDictionaryMembers(PrimitiveReader reader) {
        // //target int[][] dictionaryMembers
        int dictionaryCount = dictionaryMembers.length;
        int d = dictionaryCount;
        while (--d >= 0) {
            int h = PrimitiveReader.readIntegerUnsigned(reader);// length of reset script (eg member list)
            int[] members = new int[h];
            while (--h >= 0) {
                members[h] = PrimitiveReader.readIntegerSigned(reader);
            }
            dictionaryMembers[d] = members;
        }
    }

    /**
     * 
     * Save template scripts to the catalog file. The Script is made up of the
     * field id(s) or Tokens. Each field value needs to know the id so it is
     * stored by id. All other types (group tasks,dictionary tasks) just need to
     * be executed so they are stored as tokens only. These special tasks
     * frequently multiple tokens to a single id which requires that the token
     * is used in all cases. An example is the Open and Close tokens for a given
     * group.
     * 
     * 
     * @param writer
     * @param uniqueTemplateIds
     * @param biggestTemplateId
     * @param catalogScriptFieldNames 
     * @param scripts
     */
    private static void saveTemplateScripts(final PrimitiveWriter writer, int uniqueTemplateIds, long biggestTemplateId,
            int[] catalogScriptTokens, long[] catalogScriptFieldIds, String[] catalogScriptFieldNames, String[] dictionaryNames,
            int scriptLength, LongHashTable templateToOffset, final LongHashTable templateToLimit ) {
        // what size array will we need for template lookup. this must be a
        // power of two
        // therefore we will only store the exponent given a base of two.
        // this is not so much for making the file smaller but rather to do the
        // computation
        // now instead of at runtime when latency is an issue.

        PrimitiveWriter.writeIntegerUnsigned(scriptLength, writer);

        // total number of templates are are defining here in the catalog
        PrimitiveWriter.writeIntegerUnsigned(uniqueTemplateIds, writer);
        // write each template index
        
        LongHashTable.visit(templateToOffset, new LongHashTableVisitor() {

			@Override
			public void visit(long key, int value) {
				
				PrimitiveWriter.writeLongUnsigned(key, writer); 
				// return the index to its original value (-1)
                PrimitiveWriter.writeIntegerUnsigned(value - 1, writer);
                PrimitiveWriter.writeLongUnsigned(LongHashTable.getItem(templateToLimit, key), writer);
				
			}} );
               

        // write the scripts
        int i = scriptLength;
        while (--i >= 0) {
            PrimitiveWriter.writeIntegerSigned(catalogScriptTokens[i], writer);
            PrimitiveWriter.writeLongUnsigned(catalogScriptFieldIds[i], writer); 
           
            writeUTF8(writer, catalogScriptFieldNames[i]);
            writeUTF8(writer, dictionaryNames[i]);
            
        }
    }

	public static void writeUTF8(final PrimitiveWriter writer, String name) {
		int len = null==name?0:name.length();
		PrimitiveWriter.writeIntegerUnsigned(len, writer);
		if (len>0) {
		    PrimitiveWriter.ensureSpace(name.length(),writer);
		    
		    //convert from chars to bytes
		    //writeByteArrayData()
		    int len1 = name.length();
		    int limit = writer.limit;
		    int c = 0;
		    while (c < len1) {
		        limit = Pipe.encodeSingleChar((int) name.charAt(c++), writer.buffer, 0xFFFFFFFF, limit);
		    }
		    writer.limit = limit;
		}
	}

    public DictionaryFactory dictionaryFactory() {
        return dictionaryFactory;
    }
   
    public int maxTemplatePMapSize() {
        return maxTemplatePMapSize;
    }

    public long maxFieldId() {
        return maxFieldId;
    }

    public int[][] dictionaryResetMembers() {
        return dictionaryMembers;
    }

    public ClientConfig clientConfig() {
        return clientConfig;
    }

    public int templatesCount() {
        return templatesInCatalog;
    }

    public int[] fullScript() {
        return getScriptTokens();
    }
    public long[] fieldIdScript() {
        return scriptFieldIds;
    }
    public String[] fieldNameScript() {
        return scriptFieldNames;
    }


    public int maxNonTemplatePMapSize() {
        return maxNonTemplatePMapSize;
    }

    public int getMaxGroupDepth() {
        return maxPMapDepth;
    }

    public LongHashTable getTemplateStartIdx() {
        return templateToStartIdx;
    }

    public int[] getScriptTokens() {
        return scriptTokens;
    }

    public FieldReferenceOffsetManager getFROM() {
        return from;
    }

    public static PipeBundle buildRingBuffers(TemplateCatalogConfig catalog, byte primaryBits, byte secondaryBits) {
		return PipeBundle.buildRingBuffers(new Pipe(new PipeConfig(primaryBits, secondaryBits, catalog.ringByteConstants(), new MessageSchemaDynamic(catalog.getFROM()))).initBuffers());
	}
    
	public static FieldReferenceOffsetManager createFieldReferenceOffsetManager(TemplateCatalogConfig config) {
		
		//the scriptTokens array is too long and must be shortened, this will cause some garbage once but the length is important
		int tokenCount = 0;
		if (null!=config.scriptTokens) {//for teting
			while (tokenCount<config.scriptTokens.length && config.scriptTokens[tokenCount]<0) {
				tokenCount++;
			}		
			if (0 == tokenCount || 0 != config.scriptTokens[config.scriptTokens.length-1] ) {//this hack is here for testing
				tokenCount = config.scriptTokens.length;
			}
		}
		
		return new FieldReferenceOffsetManager(   null==config.scriptTokens? null : Arrays.copyOfRange(config.scriptTokens, 0, tokenCount), 
									        	  config.clientConfig.getPreableBytes(), 
									              config.fieldNameScript(),
									              config.fieldIdScript(),
									              config.dictionaryScript(),
									              "Catalog");
		
		
	}

	private String[] dictionaryScript() {
		return scriptDictionaryNames;
	}

	public static void load(DictionaryFactory df, PrimitiveReader reader) {
	
		df.singleBytesSize = PrimitiveReader.readIntegerUnsigned(reader);
		df.gapBytesSize = PrimitiveReader.readIntegerUnsigned(reader);
		
		df.integerCount = PrimitiveReader.readIntegerUnsigned(reader);
		df.longCount = PrimitiveReader.readIntegerUnsigned(reader);
		df.bytesCount = PrimitiveReader.readIntegerUnsigned(reader);
	
		df.integerInitCount = PrimitiveReader.readIntegerUnsigned(reader);
		df.integerInitIndex = new int[df.integerInitCount];
		df.integerInitValue = new int[df.integerInitCount];
	    int c = df.integerInitCount;
	    while (--c >= 0) {
	    	df.integerInitIndex[c] = PrimitiveReader.readIntegerUnsigned(reader);
	    	df.integerInitValue[c] = PrimitiveReader.readIntegerSigned(reader);
	    }
	
	    df.longInitCount = PrimitiveReader.readIntegerUnsigned(reader);
	    df.longInitIndex = new int[df.longInitCount];
	    df.longInitValue = new long[df.longInitCount];
	    c = df.longInitCount;
	    while (--c >= 0) {
	    	df.longInitIndex[c] = PrimitiveReader.readIntegerUnsigned(reader);
	    	df.longInitValue[c] = PrimitiveReader.readLongSigned(reader);
	    }
	
	
	    df.byteInitCount = PrimitiveReader.readIntegerUnsigned(reader);
	    df.byteInitIndex = new int[df.byteInitCount];
	    df.byteInitValue = new byte[df.byteInitCount][];
	    c = df.byteInitCount;
	    while (--c >= 0) {
	    	df.byteInitIndex[c] = PrimitiveReader.readIntegerUnsigned(reader);
	        int len = PrimitiveReader.readIntegerUnsigned(reader);
	        if (len>0) {
	        	byte[] value = new byte[len];
	        	PrimitiveReader.readByteData(value, 0, len, reader);
	        	df.byteInitValue[c] = value;
	        } else {
	        	if (len<0) {
	        		df.byteInitValue[c]=null;
	        	} else {
	        		df.byteInitValue[c]=new byte[0];
	        	}
	        }
	    }
	    df.byteInitTotalLength = PrimitiveReader.readIntegerUnsigned(reader);
	
	}

	public static void save(DictionaryFactory df, PrimitiveWriter writer) {
	
		PrimitiveWriter.writeIntegerUnsigned(df.singleBytesSize, writer);
		PrimitiveWriter.writeIntegerUnsigned(df.gapBytesSize, writer);
		
	    PrimitiveWriter.writeIntegerUnsigned(df.integerCount, writer);
	    PrimitiveWriter.writeIntegerUnsigned(df.longCount, writer);
	    PrimitiveWriter.writeIntegerUnsigned(df.bytesCount, writer);
	
	    PrimitiveWriter.writeIntegerUnsigned(df.integerInitCount, writer);
	    int c = df.integerInitCount;
	    while (--c >= 0) {
	        PrimitiveWriter.writeIntegerUnsigned(df.integerInitIndex[c], writer);
	        PrimitiveWriter.writeIntegerSigned(df.integerInitValue[c], writer);
	    }
	
	    PrimitiveWriter.writeIntegerUnsigned(df.longInitCount, writer);
	    c = df.longInitCount;
	    while (--c >= 0) {
	        PrimitiveWriter.writeIntegerUnsigned(df.longInitIndex[c], writer);
	        PrimitiveWriter.writeLongSigned(df.longInitValue[c], writer);
	    }
	
	    PrimitiveWriter.writeIntegerUnsigned(df.byteInitCount, writer);
	    c = df.byteInitCount;
	    while (--c >= 0) {
	        PrimitiveWriter.writeIntegerUnsigned(df.byteInitIndex[c], writer);
	        byte[] value = df.byteInitValue[c];
	        PrimitiveWriter.writeIntegerUnsigned(null==value? -1 :value.length, writer);
	        if (null!=value && value.length>0) {
	        	PrimitiveWriter.writeByteArrayData(value, 0, value.length, writer);
	        }
	    }
	    PrimitiveWriter.writeIntegerUnsigned(df.byteInitTotalLength, writer);
	
	    /*
	     * Fastest searialize deserialize however its more verbose and there is
	     * no object dectection and construction.
	     * 
	     * These files can be deleted and modified but those changes are only
	     * refelected on startup. New templates can be added but an explicit
	     * call must be made to load them. The new templates will be loaded
	     * dynamicaly on first use but this is not recommended.
	     */
	
	}

	public static void writeTemplateCatalog(TemplateHandler handler, int byteGap, int maxByteLength, PrimitiveWriter writer, ClientConfig clientConfig) {
	
	    TemplateHandler.postProcessDictionary(handler, byteGap, maxByteLength);
	
	   //System.err.println("Names:"+ Arrays.toString(catalogScriptFieldNames));
	
	    // write catalog data.
	    save(writer, handler.fieldIdBiggest, handler.templateIdUnique, handler.templateIdBiggest, handler.defaultConstValues,
	    		handler.catalogLargestTemplatePMap, handler.catalogLargestNonTemplatePMap, handler.tokenIdxMembers, handler.tokenIdxMemberHeads,
	    		handler.catalogScriptTokens, handler.catalogScriptFieldIds, handler.catalogScriptFieldNames, handler.catalogScriptDictionaryNames,
	    		handler.catalogTemplateScriptIdx,  handler.templateToOffset, handler.templateToLimit ,
	    		handler.maxGroupTokenStackDepth + 1, clientConfig);
	
	    // close stream.
	    PrimitiveWriter.flush(writer);
	}

	public static int maxPMapCountInBytes(TemplateCatalogConfig catalog) {
        return 2 + ((
                      catalog.maxTemplatePMapSize()>catalog.maxNonTemplatePMapSize() ?
                    		  catalog.maxTemplatePMapSize() + 2:
                    	      catalog.maxNonTemplatePMapSize() + 2) * catalog.getMaxGroupDepth());
    }

}
src/main/java/com/ociweb/jfast/stream/FASTDecoder.java
public abstract class FASTDecoder{
        
	public final LongHashTable templateStartIdx;
    //runtime count of sequence lengths
    public int sequenceCountStackHead = -1;
    public final int[] sequenceCountStack;
    
    //private ring buffers for writing content into
    public final PipeBundle ringBuffers;
    
    //dictionary data
    protected final long[] rLongDictionary; //final array with constant references
    protected final int[] rIntDictionary; //final array with constant references
    protected final LocalHeap byteHeap;
    
    public int activeScriptCursor=-1; //needed by generated code to hold state between calls.
    public int msgIdx=-1; //must hold between read (wait for space on queue) and write of templateId
    public int preambleA=0; //must hold between read (wait for space on queue) and write (if it happens)
    public int preambleB=0; //must hold between read (wait for space on queue) and write (if it happens)
    public int maxPMapCountInBytes;       
    
    public final byte[] preambleData;
   
        
    public FASTDecoder(TemplateCatalogConfig catalog) {
		this(catalog, 
             PipeBundle.buildRingBuffers(new Pipe(new PipeConfig((byte)15, (byte)7, catalog.ringByteConstants(), new MessageSchemaDynamic(catalog.getFROM()))).initBuffers()) );
        
    }
    
    public FASTDecoder(TemplateCatalogConfig catalog, PipeBundle ringBuffers) {
        this(catalog.dictionaryFactory(), 
        	 catalog.getMaxGroupDepth(),
        	 catalog.getTemplateStartIdx(), 
        	 catalog.clientConfig().getPreableBytes(), 
             ringBuffers, 
             TemplateCatalogConfig.maxPMapCountInBytes(catalog) );
        
    }
            
    private FASTDecoder(DictionaryFactory dcr, int maxNestedGroupDepth, 
    		            LongHashTable templateStartIdx,
    		            int preambleBytes,
			            PipeBundle ringBuffers, 
			            int maxPMapCountInBytes) {

        this.byteHeap = dcr.byteDictionary();
        
        this.sequenceCountStack = new int[maxNestedGroupDepth];
        this.rIntDictionary = dcr.integerDictionary();
        this.rLongDictionary = dcr.longDictionary();
        this.templateStartIdx = templateStartIdx; 
        this.preambleData = new byte[preambleBytes];
        
        this.ringBuffers = ringBuffers;
        
        this.maxPMapCountInBytes = maxPMapCountInBytes;
        
        assert (rIntDictionary.length < TokenBuilder.MAX_INSTANCE);
        assert (TokenBuilder.isPowerOfTwo(rIntDictionary.length));
        assert (rLongDictionary.length < TokenBuilder.MAX_INSTANCE);
        assert (TokenBuilder.isPowerOfTwo(rLongDictionary.length));
    }
    
    
    
    public static void reset(DictionaryFactory dictionaryFactory, FASTDecoder decoder) {
        
        // clear all previous values to un-set
        dictionaryFactory.reset(decoder.rIntDictionary); 
        dictionaryFactory.reset(decoder.rLongDictionary); 
                
        
        if (null!=decoder.byteHeap) {
            LocalHeap.reset(decoder.byteHeap);
        }
        decoder.sequenceCountStackHead = -1;
        
        PipeBundle.reset(decoder.ringBuffers);        

    }
    

    public abstract int decode(PrimitiveReader reader);


       


}
src/main/java/com/ociweb/jfast/primitive/PrimitiveReader.java
public final class PrimitiveReader {

    // Note: only the type/opp combos used will get in-lined, this small
    // footprint will fit in execution cache.
    // if we in-line too much the block will be to large and may spill.

    private final int resetLimit;  
    private final FASTInput input;
    public final byte[] buffer;
    
    private long totalReader;
    
    private byte[] invPmapStack;
    private int invPmapStackDepth;

    public int position; 
    public int limit;
    
    public int pmapIdxBitBlock = -1; //idx high in pmap data
    
    
    private InputBlockagePolicy blockagePolicy = new InputBlockagePolicy(){ //   blockagePolicy
        
        @Override
        public void detectedInputBlockage(int need, FASTInput input) {
        }

        @Override
        public void resolvedInputBlockage(FASTInput input) {
        }};
  
     //only called when we need more data and the input is not providing any
     //TODO: B, this will be refactored again as we add "reactive streams" into this implementation   
     public void setInputPolicy(InputBlockagePolicy is) {
       blockagePolicy = is; //TODO: B, this needs to be specific for this instance?
     }
        
    
	public static void printDebugData(PrimitiveReader reader) {
		int pos = reader.position;
		int lim = Math.min(reader.limit, pos+70);
		
		long base = reader.totalReader-(reader.limit-reader.position)-reader.position;
		System.err.println("printing details of bytes "+(base+pos)+" up to "+(base+lim));
		while (pos<lim) {
			
			String temp = "00000000"+Integer.toBinaryString(reader.buffer[pos]);    		
			System.err.println((base+pos)+" data:"+ temp.substring(temp.length()-8)+"    "+reader.buffer[pos] );
			
			pos++;
		}
				
				
		
	}


	/**
     * 
     * Making the bufferSize large will decrease the number of copies but may increase latency.
     * Making the bufferSize small will decrease latency but may reduce overall throughput.
     * 
     * @param bufferSizeInBytes must be large enough to hold a single group
     * @param input
     * @param maxPMapCountInBytes must be large enough to hold deepest possible nesting of pmaps
     */
    public PrimitiveReader(int bufferSizeInBytes, FASTInput input, int maxPMapCountInBytes) { 
        this.input = input;
        this.buffer = new byte[bufferSizeInBytes];
        this.resetLimit = 0;
        this.position = 0;
        this.limit = 0;        
        this.invPmapStack = new byte[maxPMapCountInBytes];//need trailing bytes to avoid conditional when using.
        this.invPmapStackDepth = maxPMapCountInBytes-2;
        input.init(this.buffer);
    }
    
    public PrimitiveReader(byte[] buffer) {
        this.input = null; //TODO: C, may want dummy impl for this.
        this.buffer = buffer;
        this.resetLimit = buffer.length;
        
        this.position = 0;
        this.limit = buffer.length;
        //in this case where the full data is provided then we know it can not be larger than the buffer.
        int maxPMapCountInBytes = buffer.length+6;
        this.invPmapStack = new byte[maxPMapCountInBytes];//need trailing bytes to avoid conditional when using.
        this.invPmapStackDepth = maxPMapCountInBytes-2;

    }
    
    public PrimitiveReader(byte[] buffer, int maxPMapCountInBytes) {
        this.input = null; //TODO: C, may want dummy impl for this.
        this.buffer = buffer;
        this.resetLimit = buffer.length;
        
        this.position = 0;
        this.limit = buffer.length;
        this.invPmapStack = new byte[maxPMapCountInBytes];//need trailing bytes to avoid conditional when using.
        this.invPmapStackDepth = maxPMapCountInBytes-2;

    }

    /**
     * Reset the internal state to its initial values.
     * 
     * This is needed when re-using the reader expcially by unit tests.
     * @param reader
     */
    public static final void reset(PrimitiveReader reader) {
        reader.totalReader = 0;
        reader.position = 0;
        reader.limit = reader.resetLimit;
        reader.pmapIdxBitBlock = -1;
        reader.invPmapStackDepth = reader.invPmapStack.length - 2;

    }
    
    /**
     * Returns the total number of bytes that have been taken from the input. 
     * 
     * @param reader
     * @return
     */
    public static final long totalRead(PrimitiveReader reader) {
        return reader.totalReader;
    }

    /**
     * Returns the total number of bytes that are currently waiting in the buffer to be parsed.
     * 
     * Using this with the totalRead method allows the caller to compute the exact byte positions being parsed.
     * 
     * @param reader
     * @return
     */
    public static final int bytesReadyToParse(PrimitiveReader reader) {
        return reader.limit - reader.position;
    }

    /**
     * Non blocking call that will pull more data from the input if the internal
     * buffer has room at the source has data available.
     * 
     * @param reader
     */
    public static final void fetch(PrimitiveReader reader) {
        fetch(0, reader);
    }
    
    /**
     *  Blocking call will not return until the need is met because the parser has
     *  determined that we can not continue until this data is provided.
     *  this call may however read in more than the need because its ready
     *  and convenient to reduce future calls.
     *  
     * @param need
     * @param reader
     */
    private static void fetch(int need, PrimitiveReader reader) {

        need = fetchAvail(need, reader);        
        if (need > 0) {     
            reader.blockagePolicy.detectedInputBlockage(need, reader.input);
            if (reader.limit+need > reader.buffer.length) {
            	throw new FASTException("buffer for PrimitiveReader is too small");
            }
            int filled = reader.input.blockingFill(reader.limit, need);
            if (filled < need) {
                throw new FASTException("Unexpected end of data.");
            }
            reader.blockagePolicy.resolvedInputBlockage(reader.input);
            reader.totalReader += filled;
            reader.limit += filled;
        }
    }
    
    private static int fetchAvail(int need, PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            reader.position = reader.limit = 0;
        }
        int remainingSpace = reader.buffer.length - reader.limit;
        if (need <= remainingSpace) {
            // fill remaining space if possible to reduce fetch later
            // but as we near the end prevent overflow by only getting what is needed.
            
            int filled = reader.input.fill(reader.limit, remainingSpace);

            reader.totalReader += filled;
            reader.limit += filled;
            //
            return need - filled;
        } else {
            return noRoomOnFetch(need, reader);
        }
    }
    
    /**
     * Move all the data in the buffer down so we have room for the new data.
     * This call must be minimized because it can cause a spike it does however allow the normal
     * case to go faster without having the deal with ring buffer logic.
     * 
     * @param need
     * @param reader
     * @return
     */
    private static int noRoomOnFetch(int need, PrimitiveReader reader) {
     //   System.err.println("____ no room on fetch need "+need+" after  pos: "+reader.position+" lim:"+reader.limit);
        
        int keepFromPosition = reader.position;
        
        // not enough room at end of buffer for the need
        int populated = reader.limit - keepFromPosition;
     //   int reqiredSize = need + populated;

//        assert (reader.buffer.length >= reqiredSize) : "internal buffer is not large enough, requres " + reqiredSize
//                + " bytes";
        
        System.arraycopy(reader.buffer, keepFromPosition, reader.buffer, 0, populated);
        // if possible fill
        int filled = reader.input.fill(populated, reader.buffer.length - populated);

        reader.position = 0;
        reader.totalReader += filled;
        reader.limit = populated + filled;
        
    //    System.err.println("new pos:"+reader.position+" lim:"+reader.limit+" filled:"+filled);

        return need - filled;

    }



    // ///////////////
    // pmapStructure
    // 1 2 3 4 5 D ? I 2 3 4 X X
    // 0 0 0 0 1 D ? I 0 0 1 X X
    //
    // D delta to last position
    // I pmapIdx of last stack frame
    // //
    // called at the start of each group unless group knows it has no pmap
    
    /**
     * Begin/Open a new PMap.  This is should be done for the begging of the messages or groups
     * that require a PMap based on the used operators within that group.
     * 
     * @param pmapMaxSize the PMap is known to never be larger than this
     * @param reader
     */
    
    public static final void openPMap(final int pmapMaxSize, PrimitiveReader reader) {

        //TODO: B, pmapMaxSize is a constant for many templates and can be injected.
        // set next bit to read
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        
        // push the old index for resume
        reader.invPmapStack[reader.invPmapStackDepth - 1] = (byte) (reader.pmapIdxBitBlock>>16);  //reader.pmapIdx;

        int k = reader.invPmapStackDepth -= (pmapMaxSize + 2);
//        if (k<0) {
//            new Exception("bad k value "+k).printStackTrace();
//            System.exit(0);
//        }

        reader.pmapIdxBitBlock = (6<<16)|(0xFF&reader.buffer[reader.position]);
        
        k = walkPMapLength(pmapMaxSize, k, reader.invPmapStack, reader, reader.buffer);
        reader.invPmapStack[k] = (byte) (3 + pmapMaxSize + (reader.invPmapStackDepth - k));


    }

    private static int walkPMapLength(final int pmapMaxSize, int k, byte[] pmapStack, PrimitiveReader reader, byte[] buffer) {
        int rp = reader.position;
        if (reader.limit - rp >= pmapMaxSize) {
            if ((pmapStack[k++] = buffer[rp++]) >= 0) {
                if ((pmapStack[k++] = buffer[rp++]) >= 0) {
                	try{
	                    do {
	                    } while ((pmapStack[k++] = buffer[rp++]) >= 0);
                	} catch (ArrayIndexOutOfBoundsException outOfBounds) {
                		throw new FASTException("No high bit set to mark the end of the PMAP");
                	}
                }
            }
            reader.position = rp;
        } else {
            k = openPMapSlow(k,reader, buffer);
        }
        return k;
    }

    private static int openPMapSlow(int k, PrimitiveReader reader, byte[] buffer) {

        // must use slow path because we are near the end of the buffer.
        while ((reader.invPmapStack[k++] = buffer[reader.position++]) >= 0) {
            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
        }
        return k;
    }

    //NOTE: for consistancy and to help with branch prediction ALWAYS check this against zero unless using brancheless
    
    /**
     * Read the next PMap bit from the open PMap. The cursor is moved forward so another call will read the next bit.
     * 
     * @param reader
     * @return
     */
    public static byte readPMapBit(PrimitiveReader reader) { 
            if (reader.pmapIdxBitBlock >= 0 ) {    
			        // Frequent, 6 out of every 7 plus the last bit block 
			        int shft = reader.pmapIdxBitBlock>>16;
			        reader.pmapIdxBitBlock -= (1<<16);                
			        return (byte) (1 & (reader.pmapIdxBitBlock >>> shft)); 
			} else {   
			    return readPMapBitNextByte(reader);               
			}
    }


	//needed for code generation to eliminate conditional to detect end of 7 bits
    /**
     * Reads the next PMap and moves to the next byte position in the PMap.
     * 
     * @param reader
     * @return
     */
    public static byte readPMapBitNextByte(PrimitiveReader reader) {
    	
    	
        if (((byte)(0xFF&reader.pmapIdxBitBlock)) < 0 ) {
            reader.pmapIdxBitBlock = (6<<16)|0x80;
            return 0;
        } else {
            int tmp = (5<<16)|(0xFF&reader.invPmapStack[++reader.invPmapStackDepth]);
            reader.pmapIdxBitBlock = tmp;                           
            return (byte) (1 & (tmp >>> 6));
        }
    }

    // called at the end of each group
    
    /**
     * Close the open PMap.  Should only call when the message or group has opened a PMap
     *   
     * @param reader
     */
    public static final void closePMap(PrimitiveReader reader) {
        byte bitBlock = reader.invPmapStack[reader.invPmapStackDepth += (reader.invPmapStack[reader.invPmapStackDepth + 1])];
        reader.pmapIdxBitBlock = (reader.invPmapStack[reader.invPmapStackDepth - 1]<<16)|bitBlock;        
    }

    // ///////////////////////////////////
    // ///////////////////////////////////
    // ///////////////////////////////////

    //recursive use of the stack turns out to be a good way to unroll this loop.
    private static long readLongSignedTail(long a, PrimitiveReader reader) {
        byte v = reader.buffer[reader.position++];
        return (v<0) ? a | (v & 0x7Fl) : readLongSignedTail((a | v) << 7,reader);
    }
    
    
    /**
     * Parse a 64 bit signed value from the buffer
     * 
     * @param reader
     * @return
     */
    public static long readLongSigned(PrimitiveReader reader) {   
        int rp = reader.position;
        if (reader.limit - rp >= 10) {// not near end so go fast.
            byte v = reader.buffer[rp];        
            reader.position = rp+1;
            long accumulator = (~((long)(((v>>6)&1)-1)))&0xFFFFFFFFFFFFFF80l; 
            return (v < 0) ? accumulator |(v & 0x7F) : readLongSignedTail((accumulator | v) << 7,reader);
        }
        return readLongSignedSlow(reader);
    }

    /**
     * Slow version of 64 bit parser that is used when the position is near the end of the buffer.
     * 
     * @param reader
     * @return
     */
    private static long readLongSignedSlow(PrimitiveReader reader) {
        // slow path
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        int v = reader.buffer[reader.position++];
        long accumulator = ((v & 0x40) == 0) ? 0 : 0xFFFFFFFFFFFFFF80l;

        while (v >= 0) { // (v & 0x80)==0) {
            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
            accumulator = (accumulator | v) << 7;
            v = reader.buffer[reader.position++];
        }
        return accumulator | (v & 0x7F);
    }

    //a lot of research went into this under 6 & 7, try this a a simple loop and re-test both approaches on Java 8
    //recursive use of the stack turns out to be a good way to unroll this loop.
    private static long readLongUnsignedTail(long a, PrimitiveReader reader) {
        byte v = reader.buffer[reader.position++];
        return (v<0) ? (a << 7) | (v & 0x7F) : readLongUnsignedTail((a<<7)|v,reader);
    }
    
    
    /**
     * Parse 64 bit unsigned value from the buffer
     * 
     * @param reader
     * @return
     */
    public static long readLongUnsigned(PrimitiveReader reader) {
        
        if (reader.limit - reader.position >= 10) {// not near end so go fast.
            byte v = reader.buffer[reader.position++];
            return (v < 0) ? (v & 0x7F) : readLongUnsignedTail(v,reader);
        }
        return readLongUnsignedSlow(reader);
    }

    private static long readLongUnsignedSlow(PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        byte v = reader.buffer[reader.position++];
        long accumulator;
        if (v >= 0) { // (v & 0x80)==0) {
            accumulator = v << 7;
        } else {
            return (v & 0x7F);
        }

        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        v = reader.buffer[reader.position++];

        while (v >= 0) { // (v & 0x80)==0) {
            accumulator = (accumulator | v) << 7;

            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
            v = reader.buffer[reader.position++];

        }
        return accumulator | (v & 0x7F);
    }  
    
    //recursive use of the stack turns out to be a good way to unroll this loop.
    private static int readIntegerSignedTail(int a, PrimitiveReader reader) {
        byte v = reader.buffer[reader.position++];
        return (v<0) ? a | (v & 0x7F) : readIntegerSignedTail((a | v) << 7,reader);
    }
    
    /**
     * Parse a 32 bit signed integer from the buffer.
     * 
     */
    public static int readIntegerSigned(PrimitiveReader reader) {
    	  if (reader.limit - reader.position >= 10) {// not near end so go fast.
		    byte v = reader.buffer[reader.position++];
		 //   int accumulator = ((v & 0x40) == 0) ? 0 : 0xFFFFFF80;         
		    int accumulator = (~(((v>>6)&1)-1))&0xFFFFFF80;  //branchless                
		    return (v < 0) ? accumulator |(v & 0x7F) : readIntegerSignedTail((accumulator | v) << 7,reader);
		}
		return readIntegerSignedSlow(reader);
    }


	private static int readIntegerSignedSlow(PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        byte v = reader.buffer[reader.position++];
        int accumulator = ((v & 0x40) == 0) ? 0 : 0xFFFFFF80;

        while (v >= 0) { // (v & 0x80)==0) {
            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
            accumulator = (accumulator | v) << 7;
            v = reader.buffer[reader.position++];
        }
        return accumulator | (v & 0x7F);
    }

    public static int readIntegerUnsigned(PrimitiveReader reader) {//Invoked 100's of millions of times, must be tight.
        int rp = reader.position;
        if (reader.limit - rp >= 5) {// not near end so go fast.
            byte v = reader.buffer[rp];
            reader.position = rp+1;
            return (v < 0) ? (v & 0x7F) : readIntegerUnsignedTail(v,reader);
        } else {
            return readIntegerUnsignedSlow(reader);
        }
    }

    //TODO: C, add overflow flag to support optional int that is outside 32 bits. Without this we dont quite match the spec.
    
    //recursive use of the stack turns out to be a good way to unroll this loop.
    private static int readIntegerUnsignedTail(int a, PrimitiveReader reader) {
        byte v = reader.buffer[reader.position++];
        return (v<0) ? (a << 7) | (v & 0x7F) : readIntegerUnsignedTail((a<<7)|v,reader);
    }
    
    private static int readIntegerUnsignedSlow(PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        byte v = reader.buffer[reader.position++];
        int accumulator;
        if (v >= 0) { // (v & 0x80)==0) {
            accumulator = v << 7;
        } else {
            return (v & 0x7F);
        }

        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        v = reader.buffer[reader.position++];

        while (v >= 0) { // (v & 0x80)==0) {
            accumulator = (accumulator | v) << 7;
            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
            v = reader.buffer[reader.position++];
        }
        return accumulator | (v & 0x7F);
    }

    public static Appendable readTextASCII(Appendable target, PrimitiveReader reader) {
        if (reader.limit - reader.position < 2) {
            fetch(2, reader);
        }

        byte v = reader.buffer[reader.position];

        if (0 == v) {
            v = reader.buffer[reader.position + 1];
            if (0x80 != (v & 0xFF)) {
                throw new UnsupportedOperationException();
            }
            // nothing to change in the target
            reader.position += 2;
        } else {
            // must use count because the base of position will be in motion.
            // however the position can not be incremented or fetch may drop
            // data.

            while (reader.buffer[reader.position] >= 0) {
                try {
                    target.append((char) (reader.buffer[reader.position]));
                } catch (IOException e) {
                    throw new FASTException(e);
                }
                reader.position++;
                if (reader.position >= reader.limit) {
                    fetch(1, reader); // CAUTION: may change value of position
                }
            }
            try {
                target.append((char) (0x7F & reader.buffer[reader.position]));
            } catch (IOException e) {
                throw new FASTException(e);
            }

            reader.position++;

        }
        return target;
    }

    public static final int readTextASCIIIntoRing(byte[] target, int targetOffset, int mask, PrimitiveReader reader) {

        int rp = reader.position;
        if (reader.limit - rp > mask) {
            byte v = reader.buffer[rp];
            if (v < 0) {//single char
                target[mask & targetOffset] = (byte) (0x7F & v);            
                reader.position=rp+1;
                return 1;
            } else {
                return (v==0) ? readTextASCIIZero(reader) : readTextASCIINormal(target, targetOffset, mask, reader, v);
            }
        }        
        return readTextASCIIIntoRingSlow(target, targetOffset, mask, reader);
    }


    private static int readTextASCIINormal(byte[] target, int targetOffset, int mask, PrimitiveReader reader, byte v) {
        int p = reader.position;
        byte[] buffer = reader.buffer;
        int idx = targetOffset;
        while (v >= 0) {
            target[mask&idx++] = (byte) (buffer[p++]);
            v= buffer[p];
        }
        target[mask&idx++] = (byte) (0x7F & v);
               // System.err.println(p-reader.position);
        reader.position = p+1;
        return idx - targetOffset;// length of string
    }

    private static int readTextASCIIZero(PrimitiveReader reader) {
        int p = reader.position;
        byte[] buffer = reader.buffer;
        byte v;
        v = buffer[p + 1];
        if (0x80 != (v & 0xFF)) {
            throw new UnsupportedOperationException();
        }
        // nothing to change in the target
        reader.position += 2;
        return 0; // zero length string
    }

    private static int readTextASCIIIntoRingSlow(byte[] target, int targetOffset, int mask, PrimitiveReader reader) {
        if (reader.limit - reader.position < 2) {
            fetch(2, reader);
        }

        byte v = reader.buffer[reader.position];

        if (0 == v) {
            v = reader.buffer[reader.position + 1];
            if (0x80 != (v & 0xFF)) {
                throw new UnsupportedOperationException();
            }
            // nothing to change in the target
            reader.position += 2;
            return 0; // zero length string
        } else {
            int idx = targetOffset;
            while (v >= 0) {
                target[mask&idx++] = (byte) (reader.buffer[reader.position++]);
                if (reader.position >= reader.limit) {
                    fetch(1, reader); // CAUTION: may change value of position
                }
                v= reader.buffer[reader.position];
            }
            target[mask&idx++] = (byte) (0x7F & v); 
            reader.position++;
            return idx - targetOffset;// length of string

        }
    }
    
    
    public static final void readByteData(byte[] target, int offset, int length, PrimitiveReader reader) {
    	    	
        // ensure all the bytes are in the buffer before calling visitor
        if (reader.limit - reader.position < length) {
            fetch(length, reader);
            
            //safe place to do sanity check
            if (reader.limit - reader.position < length) {            	
            	//System.err.println(reader.limit +"  wbl:"+reader.position+" length:"+length+"   post fetch ");
            	throw new ArrayIndexOutOfBoundsException(length);
            }
            
        }
        //System.err.println("reading length:"+length+" from "+offset+" "+target.length+"  "+reader.position);
        System.arraycopy(reader.buffer, reader.position, target, offset, length);
        reader.position += length;
    }
    
    public static final int readTextASCII(byte[] target, int targetOffset, int targetLimit, PrimitiveReader reader) {

        int countDown = targetLimit - targetOffset;
        if (reader.limit - reader.position >= countDown) {
            // System.err.println("fast");
            // must use count because the base of position will be in motion.
            // however the position can not be incremented or fetch may drop
            // data.
            int idx = targetOffset;
            while (reader.buffer[reader.position] >= 0 && --countDown >= 0) {
                target[idx++] = (byte) (reader.buffer[reader.position++]);
            }
            if (--countDown >= 0) {
                target[idx++] = (byte) (0x7F & reader.buffer[reader.position++]);
                return idx - targetOffset;// length of string
            } else {
                return targetOffset - idx;// neg length of string if hit max
            }
        } else {
            return readAsciiText2Slow(target, targetOffset, countDown, reader);
        }
    }

    private static int readAsciiText2Slow(byte[] target, int targetOffset, int countDown, PrimitiveReader reader) {
        if (reader.limit - reader.position < 2) {
            fetch(2, reader);
        }

        // must use count because the base of position will be in motion.
        // however the position can not be incremented or fetch may drop data.
        int idx = targetOffset;
        while (reader.buffer[reader.position] >= 0 && --countDown >= 0) {
            target[idx++] = (byte) (reader.buffer[reader.position++]);
            if (reader.position >= reader.limit) {
                fetch(1, reader); // CAUTION: may change value of position
            }
        }
        if (--countDown >= 0) {
            target[idx++] = (byte) (0x7F & reader.buffer[reader.position++]);
            return idx - targetOffset;// length of string
        } else {
            return targetOffset - idx;// neg length of string if hit max
        }
    }

    // keep calling while byte is >=0
    public static final byte readTextASCIIByte(PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            fetch(1, reader); // CAUTION: may change value of position
        }
        return reader.buffer[reader.position++];
    }
    
    public static final void readSkipByStop(PrimitiveReader reader) {
        if (reader.position >= reader.limit) {
            fetch(1, reader);
        }
        while (reader.buffer[reader.position++] >= 0) {
            if (reader.position >= reader.limit) {
                fetch(1, reader);
            }
        }
    }

    public static final void readSkipByLengthByt(int len, PrimitiveReader reader) {
        if (reader.limit - reader.position < len) {
            fetch(len, reader);
        }
        reader.position += len;
    }






    public static final boolean isEOF(PrimitiveReader reader) {
        if (reader.limit != reader.position) {
            return false;
        }
        if (null==reader.input) {
            return true;
        }
        fetch(0, reader);
        return reader.limit != reader.position ? false : reader.input.isEOF();
    }


    // //////////////
    // /////////

    //only needed for preamble, which is not BTW found in the spec.
    public static int readRawInt(PrimitiveReader reader) {
        if (reader.limit-reader.position <4) {
            fetch(4, reader);
        }
        
        return (((0xFF & reader.buffer[reader.position++]) << 0) | 
                ((0xFF & reader.buffer[reader.position++]) << 8) |
                ((0xFF & reader.buffer[reader.position++]) << 16) | 
                ((0xFF & reader.buffer[reader.position++]) << 24));

    }


}


Question: package com.ociweb.jfast.example;
import static com.ociweb.pronghorn.pipe.PipeReader.eqASCII;
import static com.ociweb.pronghorn.pipe.PipeReader.readASCII;
import static com.ociweb.pronghorn.pipe.PipeReader.readDataLength;
import static com.ociweb.pronghorn.pipe.PipeReader.readDecimalExponent;
import static com.ociweb.pronghorn.pipe.PipeReader.readDecimalMantissa;
import static com.ociweb.pronghorn.pipe.PipeReader.readInt;
import static com.ociweb.pronghorn.pipe.PipeReader.readLong;
import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Ignore;
import com.ociweb.jfast.catalog.loader.ClientConfig;
import com.ociweb.jfast.catalog.loader.TemplateCatalogConfig;
import com.ociweb.jfast.catalog.loader.TemplateLoader;
import com.ociweb.jfast.error.FASTException;
import com.ociweb.jfast.generator.DispatchLoader;
import com.ociweb.jfast.generator.FASTClassLoader;
import com.ociweb.jfast.primitive.PrimitiveReader;
import com.ociweb.jfast.primitive.adapter.FASTInputByteArray;
import com.ociweb.jfast.primitive.adapter.FASTInputStream;
import com.ociweb.jfast.stream.FASTDecoder;
import com.ociweb.jfast.stream.FASTReaderReactor;
import com.ociweb.pronghorn.pipe.FieldReferenceOffsetManager;
import com.ociweb.pronghorn.pipe.MessageSchemaDynamic;
import com.ociweb.pronghorn.pipe.Pipe;
import com.ociweb.pronghorn.pipe.PipeConfig;
import com.ociweb.pronghorn.pipe.PipeBundle;
import com.ociweb.pronghorn.pipe.PipeReader;





public class UsageExample {

 
    
    //TODO: AA, this is not complete and needs to be cleaned up. for a simple example of consuming the data.
    
	@Ignore //Still in work needs to assert all the right stuff.
	public void builtInTest() {
		//this example uses the preamble feature
        //large value for bandwidth, small for latency
        ClientConfig clientConfig = new ClientConfig();
        clientConfig.setPreableBytes((short)4);
        String templateSource = "/performance/example.xml";
        String dataSource = "/performance/complex30000.dat";
        boolean singleThreaded = true;
        
        decode(clientConfig, templateSource, dataSource, singleThreaded);
	}
    
    public void decode(ClientConfig clientConfig, String templateSource, String dataSource, boolean single) {
         final int count = 128;
                
                  
         //TODO: C,  for multi test we really need to have it writing to multiple ring buffers.
          byte[] catBytes = buildRawCatalogData(clientConfig, templateSource);

Answer:           TemplateCatalogConfig catalog = new TemplateCatalogConfig(catBytes);