import java.io.*; 
import java.util.*;  // for the Vector and Hashtable classes
import java.awt.*; 
import ij.*; 
import ij.plugin.*;
import ij.process.*;
import ij.io.*;
import ij.measure.*;
//import ij.IJ.*;

// ------------------------------------------
// DM3_Reader.java
// ------------------------------------------
// This plugin will read DM3 files produced by Gatan Digital Micrograph
// Decoding is based on info gleaned from the EMAN project based at Baylor
// Made a start using the Analyze_Reader plugin by Guy Williams and
// the Biorad_Reader as a base, but not that much remains.
// The guts were significantly inspired by the GatanDM3.C and
// GatanDM3.h files of the EMAN project source code:
// http://ncmi.bcm.tmc.edu/~stevel/EMAN/doc/
// ------------------------------------
// Greg Jefferis, 
// Dept Biological Sciences,
// Stanford University
// jefferis@stanford.edu
// -------------------------------------------
// as of v1.0.1 030615
// -------------------------------------------
// - reads 16 bit images only,
// - correctly parses all simple tags
// - Places tags in File/Show Info
//   including acquisition time / sample date etc.
// - allows spatial calibration of images
// - sets minimum and maximum intensity
// ------------------------------------
// as of v1.1 030615
// - went back to making it an ImagePlus extension
// ------------------------------------
// as of v1.2.6 030618
// - Corrected a bad bug - The offset for the image data was off
//   by +1 
// - AND I never told the image opener whether the image data was
//   in little endian format or not.
// - So what I was doing was reading one byte from pixel n and one from pixel n+1!
//   (which looked more or less like the correct number in big-endian)
// - Added my best guess for what ImageJ image type should be set for 
//   the standard data types 
// - Changed load(), parseDM3() and getDM3FileInfo functions to receive 
//   directory and FileName as parameters - that way load() can be called
//   directly bypassing the run() function.
// ------------------------------------
// v 1.2.7 030621
// - Most functions were previously defined as public, now restricted
//   to those that might actually get an external call
// - read through code once making a few small tidies and improving comments
// ------------------------------------
// v 1.2.8 030624
// - Improved ability to set min/max brightness of image according to information
//   in the Gatan file after bug report from Charles Daghlian at Dartmouth.
//   This takes place at the end of the load()function.
// ------------------------------------
// v 1.2.9 030625
// - Fixed handling of signed 16 bit and 32 bit images by using the 
//   FileInfo.GRAY16_SIGNED and FileInfo.GRAY32_INT constants
// & keeping the Calibration object generated by ImageJ when the image
//   is loaded rather than creating my own from scratch - I had been
//   throwing away the brightness calibration which ImageJ does on 
//   16 bit signed images as a result.
// & supplying _raw_ values of loVal and hiVal to setMinAndMax()
// ------------------------------------
// v 1.3.0 030625
// - Removed calls to functions introduced since Java 1.1 to allow
//   plugin to run on OS9 Macs with Java 1.1.7
// - However as far as I can see there is a bug in MRJ that prevents 
//   UTF-16 conversion from occurring so unicode strings in the
//   DM3 tags don't import.  Instead there will be a string saying
//   "couldn't read string blah".  This is annoying because the 
//   calibration units are given as a unicode string.
// ------------------------------------
// v 1.3.1 030625
// - Fixed UTF-16 conversion bug in MRJ by writing a quick and dirty unicode 
//   reader to be used if the UTF-16 conversion fails.  Now correctly
//   sets the spatial calibration on OS9
// - Another thing that I noticed was that it seemed to be important to
//   compile with v1.3.1 if one wanted to use the class with v1.1.7
// ------------------------------------
// v 1.3.2 030808
// - Fixed a bug in getDM3FileInfo which meant that the last rather than
//   the largest image in the DM3 file was chosen
// - Fixed a bug in getDM3CalibrationInfo which caused a failure to
//   recognise "nm" as a valid calibration unit - the problem was caused
//   by doing (unit == "nm") instead of (unit.equals("nm"))
// ------------------------------------
// v 1.3.3 030821
// - Fixed a remaining bug in the image selection routine which now
//   works for all files that I have available to test.
// ------------------------------------
// v 1.3.4 040506
// - Fixed a bug reading in USHORT image data.  The problem was that
// USHORT data and strings are hard to tell apart.  Although Image Data
// can be identified categorically, other lumps of data (e.g. LUTs)
// scattered throughout the file are harder to spot and will cause problems
// if read as a string.  Have compromised by reading as string if:
//   + it isn't image data 
//   + but is an unsigned short array 
//   + of less than 256 bytes 
// ------------------------------------
// v 1.3.4 051213 (Yes the 2nd v1.3.4 - hadn't noticed a branch!)
// - Fixed a bug which prevented units other than m or nm being passed to
//   calibration object.  Occasioned by a file with units 1/nm
// ------------------------------------
// v 1.3.5 051213
// - Adding handling of DIFFRACTION mode images (ie reciprocal space) by
//   setting FHT property and copying 
//   calibration object.  Occasioned by a file with units 1/nm
// ------------------------------------
// v 1.3.6 060904
// - Added storing of struct fields to tag list
//   They are displayed as tag= {1,2,3,4}
//   This means that Digital Micrograph selection rectangles
//   can now be identified from Show Info
// ------------------------------------
// v 1.3.7 070831
// - debugLevel is now set according to IJ.debugMode
//   (accesible from Edit ... Options ... Misc)
// - Can now open data type 23 (RGBA_UINT8_3_DATA) images
// ------------------------------------
// v 1.3.8 080326
// - Fixed a bug in which tag hashes were not cleared when reading a new file
// - Small speed improvement by converting tags to String via StringBuffer
// - Both thanks to report by Eric Olson at UIUC
     
public class DM3_Reader extends ImagePlus implements PlugIn 
{
	// Decide whether to use Gatan's information for determining the min
	// and maximum brightness thresholds for display or leave to ImageJ
	// I find Gatan more reliable
	public boolean useGatanMinMax = true;

	private boolean littleEndian = true;  // default for .dm3 files
	// nb all tags are written big-endian, it is only the actual data 
	// attached to each tag that may be little-endian (and will be for PC files)
	//private String directory;
	//private String fileName;
	private RandomAccessFile f;  // This stream will be used for reading by parseDM3()

	private FileInfo fi;

	private String notes = "";  // I will store interesting file info in here 
	
	//  0=none, 1-3=basic, 4-5=simple, 6-10 verbose
	private static final int debugLevel = IJ.debugMode?10:0; 

	// the number of the chosen image in the DM3 file
	// since there apparently may be several - usually at least a thumbnail
	// I will select the largest image.  If there are multiple images
	// of the same size, the first will be chosen.
	private int chosenImage = 1;
	
	private int curGroupLevel=-1;  // Track how deep is the group we are currently reading
	private static final int MAXDEPTH = 64; // Maximum number of levels of tags
	private int[] curGroupAtLevelX=new int[MAXDEPTH];  // To track group at current level
	private String[] curGroupNameAtLevelX=new String[MAXDEPTH];  // To track group name at current level

	private int[] curTagAtLevelX=new int[MAXDEPTH];  // To track tag number at current level
	private String curTagName = "";  // the name of the current tag data item

	// Will use these to store tags
	private Vector storedTags = new Vector();
	private Hashtable tagHash = new Hashtable();
	
	// Set up constants for the different encoded data types used in DM3 files
	private static final int SHORT   = 2;
	private static final int LONG    = 3;
	private static final int USHORT  = 4;
	private static final int ULONG   = 5;
	private static final int FLOAT   = 6;
	private static final int DOUBLE  = 7;
	private static final int BOOLEAN = 8;
	private static final int CHAR    = 9;
	private static final int OCTET   = 10;
	private static final int STRUCT  = 15;
	private static final int STRING  = 18;
	private static final int ARRAY   = 20;
		
	// This the lhs of Image list tags
	private static final String IMGLIST = "root.ImageList.";
	// This is the lhs for Document Object List Tags
	// root.DocumentObjectList.0.AnnotationGroupList.0.AnnotationType = 31
	private static final String OBJLIST = "root.DocumentObjectList.";

	public void run(String arg)  {
		String directory = "";
		String fileName = arg;
		//if (debugLevel>5);
	
		if (debugLevel>5) IJ.write("IN:dir = "+directory+", file="+fileName);
		if ((arg==null) || (arg==""))
		{	// Choose a file since none specified
			OpenDialog od = new OpenDialog("Load DM3 File...", arg);
			fileName = od.getFileName();
			if (fileName==null)
			  return;
			directory = od.getDirectory();
			if (debugLevel>5) IJ.write("IF:dir = "+directory+", file="+fileName);
		}
		else
		{	// we were sent a filename to open
			File dest = new File(arg);
			directory = dest.getParent();
			fileName = dest.getName();
			if (debugLevel>5) IJ.write("ELSE:dir = "+directory+", file="+fileName);			
		}

		// Load in the image
		ImagePlus imp = load(directory, fileName);
		if (imp==null) return;
		
		// Attach the Image Processor
		setProcessor(fileName, imp.getProcessor());
		// Copy the scale info over
		copyScale(imp);
		// Copy the Show Info field over
		setProperty("Info",imp.getProperty("Info"));
		// and the FHT property (to handle diffraction mode images)
		if(imp.getProperty("FHT")!=null) setProperty("FHT", imp.getProperty("FHT"));
		
		// Show the image if it was selected by the file
		// chooser, don't if an argument was passed ie
		// some other ImageJ process called the plugin
		if (arg.equals("")) show();
	}

	public ImagePlus load(String directory, String fileName) /*throws IOException*/ {
    
		if ((fileName == null) || (fileName == "")) return null;

		if (!directory.endsWith(File.separator)) directory += File.separator;

		IJ.showStatus("Loading DM3 File: " + directory + fileName);

		// Clear the lists of tags in which additional info will be stored
		tagHash.clear();
		storedTags.clear();
		// Try calling the parse routine
		try{ parseDM3(directory, fileName);}
		catch (Exception e) {
			IJ.showStatus("parseDM3() error");
			IJ.showMessage("DM3_Reader", ""+e);
			return null;
		}
		
		// Make a blank file information object
		fi = new FileInfo();
		
		// Go and fetch the DM3 specific file Information
		try {fi=getDM3FileInfo(directory, fileName);}
		// This is in case of trouble parsing the tag table
		catch (Exception e) {
			IJ.showStatus("");
			IJ.showMessage("DM3_Reader", "gDM3:"+e);
			return null;
		}
		
		// Write out Calculated Offset if reqd
		if(debugLevel>1) IJ.write("Calculated offset = "+fi.offset);
		if(debugLevel>1) IJ.write("Chosen image = "+chosenImage);

		// Open the image!
		FileOpener fo = new FileOpener(fi);  
		ImagePlus imp = fo.open(false);
		//if(debugLevel>5) if(imp==null) IJ.write("Image load failed!");
		
		// Write out the contents of the storedTags list
		// and set the value of notes
		StringBuffer notesBuffer=new StringBuffer();
		for (int i = 0; i<storedTags.size();i++){
			// Can decide whether I want to do this
			//IJ.log((String) storedTags.elementAt(i));
			notesBuffer.append( (String) storedTags.elementAt(i) + "\n");
		}
		notes=notesBuffer.toString();
		if (!notes.equals("")) imp.setProperty("Info", notes);
		
		// Set (spatial) calibration
		// nb pass the current calibration in case that contains useful info 
		// already (such as a brightness calibration)
		try {
			imp.setCalibration(getDM3CalibrationInfo(imp.getCalibration()));
		}
		catch (Exception e) {
			IJ.showStatus("No Calibration info in "+fileName);
		}
		// If this is a diffraction (ie reciprocal space) image then set the
		// FHT property so that ImageJ displays inverse scale
		String imagingMode = (String) tagHash.get(IMGLIST+chosenImage+".ImageTags.Microscope Info.Imaging Mode");
		if (imagingMode!=null && imagingMode.toUpperCase().equals("DIFFRACTION")){
				imp.setProperty("FHT", "Dummy FHT");
		}
		
		
		// Set the min and max brightness for display purposes 
		// from DM3 header info if required - ImageJ can do this
		// but is less robust
		if(useGatanMinMax) {
			// now searches through all tags 
			// after bug report by <Charles.P.Daghlian@Dartmouth.EDU>
			double hiVal=0.0, loVal=0.0;
			// Iterate over components of the taglist
			// looking for the image brightness tag
			// all this because I don't know how to partially match a hash key
			for (Enumeration e = tagHash.keys() ; e.hasMoreElements() ;) {
				String thisElementString = (String) e.nextElement();
				
				if( (thisElementString).endsWith("ImageDisplayInfo.HighLimit"))
					hiVal = ((Float) tagHash.get(thisElementString)).doubleValue();
				if( (thisElementString).endsWith("ImageDisplayInfo.LowLimit"))
					loVal = ((Float) tagHash.get(thisElementString)).doubleValue();
			}

			// If we found at least one, then set the min max brightness
			if (hiVal!=0.0 || loVal!=0.0) {
				// min,max are set through the image processor, so get it
				ImageProcessor ip = imp.getProcessor();
				// set them - nb setMinMax expects raw pixel values
				// if a brightness calibration is in force then getRawValue()
				// does the appropriate conversion
				ip.setMinAndMax(imp.getCalibration().getRawValue(loVal),imp.getCalibration().getRawValue(hiVal));
			}
		}
		
		return imp; 		
    }

	void parseDM3(String directory, String fileName) throws IOException {
		// This reads through the DM3 file, extracting useful tags
		// which allow one to determine the data offset etc.
		
		// alternative way to read from file - allows seeks!
		// and therefore keeps track of position (use long getFilePointer())
		// also has DataInput Interface allowing
		// reading of specific types
		f = new RandomAccessFile(directory+fileName,"r");
		if(debugLevel>0) IJ.write("Directory = "+directory);
		if(debugLevel>0) IJ.write("File = "+fileName);

		// Get the first 3 4byte ints from Header to find out
		// FileVersion (which must be 3)
		int fileVersion = f.readInt();
		if (fileVersion!=3) throw new IOException("This does not seem to be a DM3 file");
		
		if(debugLevel>5) IJ.write("File Version"+fileVersion);

		// ... file size 
		int FileSize=f.readInt();
		int lE=f.readInt();

		if(debugLevel>5) IJ.write("lE "+lE);

		// ... and whether it was written in little endian (PC) format or not
		// (Mac and Java output are big endian)
		if(lE==1) {
			littleEndian=true;
		}
		else {
			if(lE==0) littleEndian=false;
			else {
				throw new IOException("This does not seem to be a DM3 file");
			}
		}
					
		// The DM3 file has an unnamed root group which contains everything in the file
		curGroupNameAtLevelX[0] = "root";  // Set the name of the root group

		// Now go read it (and all of its sub groups.
		readTagGroup();
		
		// Close the input stream
    	f.close();
	}

	FileInfo getDM3FileInfo(String directory, String fileName) throws IOException {
		// this gets the basic file information using the contents of the tag
		// tables created by parseDM3()

		// Set the basic file information
		FileInfo fi = new FileInfo();
		fi.fileFormat = fi.RAW;
		fi.fileName = fileName;
		fi.directory = directory;
		// Originally forgot to do this - tells ImageJ what endian form the actual 
		// image data is in
		fi.intelByteOrder=littleEndian;

		chosenImage = 0;
		// Look for largest Image and assume that is the one we want
		int i=0;  // nb the first image is image = 0
		long largestDataSizeSoFar=0;
		
		// Iterate over images keeping a note of the largest image so far
		while (true) {
			// The specific part of the key we are looking for
			String rString=".ImageData.Data.Size";
			if(debugLevel>1) IJ.write("Looking for:"+IMGLIST+i+rString);
			
 			// Can we find information for image i
			if(tagHash.containsKey(IMGLIST+i+rString)) {
				if(debugLevel>1) IJ.write("Found:"+IMGLIST+i+rString);
				// how big is this image?
				long dataSize = ((Long) tagHash.get(IMGLIST+i+rString)).longValue();
				if(debugLevel>1) IJ.write("Current Data Size"+dataSize);
				
				// Is it the largest so far?
				if(dataSize>largestDataSizeSoFar) {
					// Choose this image
					largestDataSizeSoFar=dataSize;
					if(debugLevel>1) IJ.write("New Largest Data Size:"+largestDataSizeSoFar);
					chosenImage=i;
					if(debugLevel>1) IJ.write("New Chosen Image:"+chosenImage);
				}
				
				i++; // move on to the next image
			} else {
				break;  // we ran out of images
			}
		}
	
		/* Here are the ImageData.DataType definitions from GatanDM3.h
				class DataType {
		public:
					enum Type {
					0=	NULL_DATA,
					1=	SIGNED_INT16_DATA,
					...	REAL4_DATA,
						COMPLEX8_DATA,
						OBSELETE_DATA,
						PACKED_DATA,
						UNSIGNED_INT8_DATA,
						SIGNED_INT32_DATA,
						RGB_DATA,
						SIGNED_INT8_DATA,
						UNSIGNED_INT16_DATA,
						UNSIGNED_INT32_DATA,
						REAL8_DATA,
						COMPLEX16_DATA,
						BINARY_DATA,
						RGB_UINT8_0_DATA,
						RGB_UINT8_1_DATA,
						RGB_UINT16_DATA,
						RGB_FLOAT32_DATA,
						RGB_FLOAT64_DATA,
						RGBA_UINT8_0_DATA,
						RGBA_UINT8_1_DATA,
						RGBA_UINT8_2_DATA,
						RGBA_UINT8_3_DATA,
						RGBA_UINT16_DATA,
						RGBA_FLOAT32_DATA,
						RGBA_FLOAT64_DATA,
						POINT2_SINT16_0_DATA,
						POINT2_SINT16_1_DATA,
						POINT2_SINT32_0_DATA,
						POINT2_FLOAT32_0_DATA,
						RECT_SINT16_1_DATA,
						RECT_SINT32_1_DATA,
						RECT_FLOAT32_1_DATA,
						RECT_FLOAT32_0_DATA,
						SIGNED_INT64_DATA,
						UNSIGNED_INT64_DATA,
						LAST_DATA
					};
		
				};
		*/		
		// OK pick the DataType
		int dataType = ((Integer) tagHash.get(IMGLIST+chosenImage+".ImageData.DataType")).intValue();

		// I have made my best guess for types 1-14
		// ie SIGNED_INT16_DATA to BINARY_DATA
		// but I don't know how to implement the remainder 
		switch(dataType){
			case 1: //	SIGNED_INT16_DATA 
				fi.fileType=FileInfo.GRAY16_SIGNED;
				break;
			case 10: //	UNSIGNED_INT16_DATA 
				fi.fileType=FileInfo.GRAY16_UNSIGNED;
				break;
			
			case 2: // REAL4_DATA
				fi.fileType=FileInfo.GRAY32_FLOAT;
				break;
				
			//case 9:	// or SIGNED_INT8_DATA
			// NB ImageJ only handles unsigned ints - initially was treating
			// these as unsigned, but in the end decided to remove for safety

			case 6: // UNSIGNED_INT8_DATA
				fi.fileType=FileInfo.GRAY8;
				break;
			
			case 7: // SIGNED_INT32_DATA
				fi.fileType=FileInfo.GRAY32_INT;
				break;
			case 11: // UNSIGNED_INT32_DATA
				fi.fileType=FileInfo.GRAY32_UNSIGNED;
				break;
			
			case 8: // RGB_DATA
				fi.fileType=FileInfo.RGB;
				break;
			
			case 14: // BINARY_DATA
				fi.fileType=FileInfo.BITMAP;
				break;
			case 23: // RGBA_UINT8_3_DATA
				// NB it is uncertain if this data type corresponds exactly to ImageJ's ARGB
				// A definitely comes first but the RGB values could be scrambled
				// (since they were all equal on my test image)
				fi.fileType=FileInfo.ARGB;
				break;
			
			default:
				throw new IOException("Unimplemented ImageData dataType="+dataType+" in DM3 file.  See getDM3FileInfo() for details");
		}

		// Get the dimensions of the image for the chosen image
		// I'm assuming they are ordered width then height
		fi.width = ((Integer) tagHash.get(IMGLIST+chosenImage+".ImageData.Dimensions.0")).intValue();
		fi.height = ((Integer) tagHash.get(IMGLIST+chosenImage+".ImageData.Dimensions.1")).intValue();
		
		// Get the offset of the Image Data for chosen image
		fi.offset = ((Long) tagHash.get(IMGLIST+chosenImage+".ImageData.Data.Offset")).intValue();

		return fi;
	}		

	Calibration getDM3CalibrationInfo(Calibration cal){
		// get the spatial calibration information
		// could also do brightness 
		// (actually a calibration fn is applied by ImageJ according
		// to the image Type - GRAY16_SIGNED has 32768 removed in calibration

		// Figure out what the units are - need to check if nm is correct and
		// if other units are likely
		// also will m get corrupted? may be necessary to do a unicode comparison
		String unit = (String) tagHash.get(IMGLIST+chosenImage+".ImageData.Calibrations.Dimension.0.Units");

		// Reciprocal space images - return the original unit - reciprocal
		// space will be handled by setting the FHT image property		
		if (unit.startsWith("1/")) unit=unit.substring(2);

		if (unit.equals("m")){
			cal.setUnit("micron");
		} else {
			cal.setUnit(unit);
		}
		if(debugLevel>0) IJ.write("Calibration unit: "+unit);
		
		cal.pixelWidth = ((Float) tagHash.get(IMGLIST+chosenImage+".ImageData.Calibrations.Dimension.0.Scale")).doubleValue();
		cal.pixelHeight = ((Float) tagHash.get(IMGLIST+chosenImage+".ImageData.Calibrations.Dimension.1.Scale")).doubleValue();
		// not yet implemented stacks - if they exist
		//cal.pixelDepth = 
		return cal;
	}
	
	int readTagGroup()  throws IOException {
		curGroupLevel+=1;  // Go down a level since this is a new group
		curGroupAtLevelX[curGroupLevel]++;  // Increment the group counter at this level
		// Set the number of current tag at this level to -1
		// since the readTagEntry routine pre-increments
		// ie the first tag will be labelled tag 0
		curTagAtLevelX[curGroupLevel]=-1;  
		
		if(debugLevel>5) IJ.write("rTG: Current Group Level: "+curGroupLevel);
		
		int isSorted=f.readByte();
		int isOpen=f.readByte();
		int nTags=f.readInt();

		if(debugLevel>5) IJ.write("rTG: Iterating over the "+nTags+" tag entries in this group");
		// Iterate over the number of Tag Entries in this group
		for( int i = 0; i<nTags;i++) {
			readTagEntry();
		}

		// Go back up a level now that we've finished reading this group
		curGroupLevel-=1;
		
		return 1;
	};

	String makeGroupString() {
		// Produces a string which is the concatenation of the current group levels
		String tString = new String(""+curGroupAtLevelX[0]);

		for (int i=1; i<=curGroupLevel;i++){
			tString += "."+curGroupAtLevelX[i];		
		}
		
		return tString;
	}
	
	int readTagEntry() throws IOException {
		int isData=f.readByte();

		// Record that we've found a new tag at this level
		curTagAtLevelX[curGroupLevel]++;

		//Get the tag label if one exists
		int lenTagLabel=f.readShort();
		String tagLabel;
		if(lenTagLabel!=0){
			tagLabel=readString(lenTagLabel);
		} else {
			tagLabel=new String(""+curTagAtLevelX[curGroupLevel]);
		}
		
		// For debugging
		if(debugLevel>5) {
			IJ.write(curGroupLevel+"|"+makeGroupString()+": Tag label = "+tagLabel);
		} else if (debugLevel>0){
			IJ.write(curGroupLevel+": Tag label = "+tagLabel);
		}
				
		//  Figure out if the tag was data or a new group
		if (isData==21) {
			// this tag entry is data

			// OK settle what this piece of data will be called
			curTagName = new String(makeGroupNameString()+"."+tagLabel);

			// now get it
			readTagType();
		} else {
			//this tag entry is a tag group

			// Slightly ugly that this can't be done in readTagGroup
			curGroupNameAtLevelX[curGroupLevel+1]=tagLabel;  // Store the name of the group at the new level
			readTagGroup();  // which will actually increment curGroupLevel
		}
		return 1;
	};

	String makeGroupNameString() {
		// A utility function: 
		// Produces a string which is the concatenation of the current group names
		String tString = new String(curGroupNameAtLevelX[0]);

		for (int i=1; i<=curGroupLevel;i++){
			tString += "."+curGroupNameAtLevelX[i];
		}

		return tString;
	}

	int readTagType() throws IOException {
		int Delim=f.readInt();
		// Should always start with %%%%
		if (Delim!=0x25252525) throw new IOException("Tag Type delimiter not %%%%");
		
		// This is redundant info, so just ignore it.
		int nInTag=f.readInt();

		readAnyData();
		
		return 1;
	};
	
	int readAnyData() throws IOException {
	// Higher level function which dispatches to functions
	// handling specific data types
		
		// This specifies what kind of type we are dealing with
		// eg short, long, struct, array etc.
		int encodedType = f.readInt();

		// Figure out the size of the encodedType
		int etSize = encodedTypeSize(encodedType);
		if(debugLevel>5) IJ.write("rAnD, "+hexPosition()+": Tag Type = "+encodedType+", Tag Size = "+etSize);

		if(etSize>0){
			// must be a regular data type, so read it and store a tag for ir
			storeTag( curTagName,readNativeData(encodedType,etSize) );
		}
		// OK then, perhaps it's an array, struct or string.
		else if (encodedType==STRING) // String
		{
			// nb readStringData will also store tags internally
			int stringSize = f.readInt();
			readStringData(stringSize);
		}
		else if (encodedType==STRUCT) // Struct
		{
			// This now stores fields (in curly braces) but does not store 
			// field names.  In fact the code will be break for non-zero
			// field names.
			Vector structTypes = readStructTypes();
			readStructData(structTypes);
		}
		else if (encodedType==ARRAY) // Array
		{
			// This only stores a tag which I defined myself
			// to indicate the size of data chunks that are skipped
			Vector arrayTypes=readArrayTypes();
			readArrayData(arrayTypes);
		}
		else {
			throw new IOException("rAnD, 0x"+hexPosition()+": Can't understand encoded type");
		}
		return 1;
	}
	
	Object readNativeData(int encodedType,int etSize) throws IOException {
	// Does the actual reading of ordinary data types

		// since it starts as an object, it is not tied to a particular
		// data type
		Object val=null;

		if(encodedType==SHORT){ //short
			val = new Short(blreadShort());
		}
		else if(encodedType==LONG){ //long
			val = new Integer(blreadInt());
		}
		else if(encodedType==USHORT){ //u short
			val = new Short(blreadUShort());
		}
		else if(encodedType==ULONG){ //u long
			val = new Integer(blreadInt());
		}
		else if(encodedType==FLOAT){ //float
			val = new Float(blreadFloat());
		}
		else if(encodedType==DOUBLE){ //double
			val = new Double(blreadDouble());
		}
		else if(encodedType==BOOLEAN){ //boolean
			if (f.readByte()==0){
				val = new Boolean(false);
			} else {
				val = new Boolean(true);
			}
		}
		else if(encodedType==CHAR){ //char
			val = new Character( (char) f.readByte() );
		}
		else if(encodedType==OCTET){ //octet
			// what's the difference?
			val = new Byte( f.readByte() );
		} else {
			// Not a known data type
			throw new IOException("rND, 0x"+hexPosition()+": Unknown data type "+encodedType);
		}

		// Print out the value if necessary
		if(debugLevel>3){
			IJ.write("rND, 0x"+hexPosition()+": "+val);
		} else if(debugLevel>0) {
			IJ.write(""+val);
		}

		return val;
	}
	String readStringData(int stringSize) throws IOException {
	// Does the actual reading of string data types
	// These should be written as Unicode which can be directly
	// converted by the String constructor
		if(stringSize<=0) return new String("");

		// Read the string data into a temporary byte buffer.
		byte[] temp = new byte[stringSize];
		f.read(temp,0,stringSize);

		// Now convert these unicode bytes into a real string
		String rString;
		
		// Note that I can't get UTF encoding to work on MRJ 2.2.5 =
		// JDK 1.1.7 even though I think it should
		// so I have put together a kludge 
		if(littleEndian){
			// Use UTF-16LE encoding (this seems to fail with MacOS 9 Java 1.1.7)
			try {
				rString=new String(temp,"UTF-16LE");
			}
			catch (Exception e) {
				// Manual conversion of the string if the above fails
				rString="";
				for(int i=0;i<stringSize;i+=2) {
					rString+=new Character((char)((temp[i+1]&0xFF) <<8 | (temp[i] & 0xFF)));
				}
			}
			
		}else{
			// use UTF-16BE encoding (this seems to fail with MacOS 9 Java 1.1.7)
			try {
				rString=new String(temp,"UTF-16BE");
			}
			catch (Exception e) {
				// Manual conversion of the string if the above fails
				rString="";
				for(int i=0;i<stringSize;i+=2) {
					rString+=new Character((char)((temp[i]&0xFF) <<8 | (temp[i+1] & 0xFF)));
				}
			}
			
		}

		if(debugLevel>0) IJ.write("StringVal: "+rString);

		// Store the value of this tag
		storeTag(curTagName,rString);
		
		return rString;
	}
	
	Vector readArrayTypes() throws IOException {
		// Figures out the data types in an array data type
		// complicated by the fact that the array type could be a struct!

		// Don't know if this will behave for arrays of strings or arrays

		int arrayType=f.readInt();

		Vector itemTypes = new Vector();
		if (arrayType==STRUCT) {  // ie a Struct
			itemTypes = readStructTypes();
		}
		else if (arrayType==ARRAY) { // ie a sub array
			itemTypes = readArrayTypes();
			// not sure if this will work!
		}
		else {
			// assume its an array of simple types
			// add changed to addElement for Java 1.1.7 compatibility
			itemTypes.addElement(new Integer(arrayType));
		}

		return itemTypes;
	}
	int readArrayData(Vector arrayTypes) throws IOException {
		// Reads in array data

		// First thing to do is get number of array elements
		int arraySize=f.readInt();
		if(debugLevel>3) IJ.write("rArD, 0x"+hexPosition()+": Reading array of size = "+arraySize);

		// Now figure out the total width of each element in the array
		// nb these elements can have subelements if the array is an array
		// of STRUCTS etc.
		int itemSize=0;
		
		// Iterate over every type in arrayTypes - there may be more than
		// one type if this is an array of structs or arrays.
		int encodedType=0;
		for (int i = 0; i < arrayTypes.size(); i++) {
			// cast the ith Vector element to Integer and then get a regular int
			encodedType = ((Integer) arrayTypes.elementAt(i)).intValue();
			int etSize=encodedTypeSize(encodedType);
			// Now add that size to our running total
			itemSize+=etSize;
			// For debugging
			if(debugLevel>5) IJ.write("rArD: Tag Type = "+encodedType+", Tag Size = "+etSize);
			//readNativeData(encodedType,etSize);
		}
		if(debugLevel>5) IJ.write("rArD: Array Item Size = "+itemSize);

		// OK now figure out what to do with this array
		// this would be the buffer size needed to accommodate ot
		long bufSize = (long) arraySize * (long) itemSize;

		// If this isn't image data but is an unsigned short array 
		// of less than 256 bytes then it is probably a string
		if(!curTagName.endsWith("ImageData.Data") && arrayTypes.size() == 1 
		   && encodedType == USHORT &&  arraySize <256) {
			// read in as string
			String val = readStringData((int) bufSize);
		}
		else {  // treat as binary data
			// Make up my own tags to indicate data size
			storeTag(curTagName+".Size",new Long(bufSize));
			// and current offset 
			// nb for a while I had offset + 1but this was wrong!
			// and gave a peculiar staircase histogram because what I had
			// ended up doing was reading one byte each from a pair of pixels 
			// rather than 2 bytes from a single pixel.  Ugh!
			storeTag(curTagName+".Offset",new Long(f.getFilePointer()));
				
			// then go ahead and skip bufSize bytes from current position
			// without trying to read this data
			f.seek( f.getFilePointer()+bufSize);
		}
		return 1;
	}
	
	Vector readStructTypes() throws IOException {
	// Figures out the data types in a struct
		if(debugLevel>3) IJ.write("Reading Struct Types at Pos = "+f.getFilePointer()+", 0x"+hexPosition());		
		
		// nb GatanDM3 has longs - I think C++ long = 4 bytes, so use Java int
		int structNameLength=f.readInt();
		int nFields=f.readInt();

		if(debugLevel>5) IJ.write("nFields = "+nFields);

		if (nFields>100) throw new IOException("Too many fields");
		
		Vector fieldTypes = new Vector();
		int nameLength = 0;
		for (int i = 0; i<nFields; i++) {
			nameLength=f.readInt();
			if(debugLevel>10) IJ.write(i+"th namelength = "+nameLength);
			int fieldType=f.readInt();
			
			// add changed to addElement for Java 1.1.7 compatibility
		    fieldTypes.addElement(new Integer(fieldType));
		}
		
		return fieldTypes;
	}
	
	int readStructData(Vector structTypes) throws IOException {
	// Reads in struct data based on the type info in structTypes
		String structAsString="";
		
		for (int i = 0; i < structTypes.size(); i++) {
			Integer iTagType = (Integer) structTypes.elementAt(i);
			
			int encodedType = iTagType.intValue();
			int etSize=encodedTypeSize(encodedType);
			// For debugging
			if(debugLevel>5) IJ.write("Tag Type = "+encodedType+", Tag Size = "+etSize);

			// OK now get the data
			structAsString+=readNativeData(encodedType,etSize);
			// Add a comma to separate values unless this is the last entry
			if(i+1!=structTypes.size()) structAsString+=",";
		}
		storeTag(curTagName,"{"+structAsString+"}");
		return 1;
	}
	
	
	int encodedTypeSize(int encodedType){
	// returns the size in bytes of the data type
	// 030614 Replaced type numbers based on GatanDM3.h

		// so -1 will be the value returned for an unrecognised type
		// (which could include ARRAYs, STRUCTs, STRINGs)
		int width=-1;
		
		switch(encodedType){
			case 0: // blank field? Do I need this?
			width = 0; break;
			
			case BOOLEAN: //	boolean: data size = 1 
			case CHAR: //	char: data size = 1 
			case OCTET: // 	octet: data size = 1 
			width=1;		break;
			
			case SHORT: // 		1.	short: data size = 2 
			case USHORT: //		3.	unsigned short: data size = 2 
			width=2;		break;
			
			case LONG: // 2.	long: data size = 4 
			case ULONG: // 4.	unsigned long: data size = 4 
			case FLOAT: // 5.	float: data size = 4 
			width=4;		break;
				
			case DOUBLE: //	double: data size = 8 
			width=8; break;
		}
		return(width);
	}

	// Store the value and key of the tag that we have just
	// read in a table
	void storeTag( String tagName, Object tagValue){
		// add changed to addElement for Java 1.1.7 compatibility
		storedTags.addElement(new String(tagName+" = "+tagValue));
		tagHash.put(tagName,tagValue);
	}

	
	// ********************************************************	
	// the bl methods will check value of littleEndian and read
	// from the RandomAccessFile f accordingly.
	// (bl for big/little - ie can cope with either endian format)
	// ********************************************************	
  
	short blreadShort() throws IOException
	{
		if (!littleEndian) return f.readShort(); 
		byte b1 = f.readByte();
		byte b2 = f.readByte();
		return ( (short) (((b2 & 0xff) << 8) | (b1 & 0xff)) );
	}
	short blreadUShort() throws IOException
	// Identical to blreadShort - is this correct?
	// not really - java uses signed shorts, but I think it's
	// too complicated to try and implement unsigned
	// In principle, one should add 65536 to negative values
	// to convert, but then they would have to be stored as 4 byte ints
	// or something.
	{
		if (!littleEndian) return (short) f.readUnsignedShort(); 
		byte b1 = f.readByte();
		byte b2 = f.readByte();
		return ( (short) (((b2 & 0xff) << 8) | (b1 & 0xff)) );
	}

	int blreadInt() throws IOException 
	{
		if (!littleEndian) return f.readInt(); 
		byte b1 = f.readByte();
		byte b2 = f.readByte();
		byte b3 = f.readByte();
		byte b4 = f.readByte();
		return ( (((b4 & 0xff) << 24) | ((b3 & 0xff) << 16) | ((b2 & 0xff) << 8) | (b1 & 0xff)) );
	}

	long blreadLong() throws IOException
	// New fn, not quite sure if the little endian version is correct
	// OK Corrected now as far as I can tell
	{
		if (!littleEndian) return f.readLong(); 
		int i1=blreadInt();
		int i2=blreadInt();

		// need to convert intermediates to long explicitly since the standard
		// is presumably just to work with them as int
		return ( ((long) (i2 & 0xffffffff) << 32) | (long) (i1 & 0xffffffff) );
		
	}
	
	double blreadDouble() throws IOException 
	// new fn to read 8 byte doubles using blreadLong as a base
	{
		if (!littleEndian) return f.readDouble();  
		
		long orig = blreadLong();
		return (Double.longBitsToDouble(orig));
	}
	
	float blreadFloat() throws IOException 
	{
		if (!littleEndian) return f.readFloat();  

		int orig = blreadInt();
		return (Float.intBitsToFloat(orig));
	}

	// used to read in field labels
	String readString(int n) throws IOException{
		// not sure if this readString limit is sensible or necessary
		if(n>2000) throw new IOException("Can't handle strings longer than 2000 chars, n = "+n+" at pos = "+f.getFilePointer());
		
		byte[] temp = new byte[n];
		f.read(temp,0,n);

		return new String(temp);
		
	}

	String hexPosition() throws IOException {
	//  Utility fn to return current file position in hex
		return (  Long.toHexString( f.getFilePointer() )  );	
	}

}