package base;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.lang.reflect.Array;

import ncsa.hdf.hdf5lib.H5;

import oneLiners.DataConvertPureJava;

import ncsa.hdf.hdf5lib.*;
import ncsa.hdf.hdf5lib.exceptions.*;

/**
 * Hold n-dimensional data of many basic types.
 * 
 * Provides arrays for coordinates along each dimension. Provides writing and
 * reading from HDF5 files.
 * 
 */
public class DataMatrix {

	/**
	 * Maximum data length (string matrix) over which to issue a warning (128MB)
	 */
	public static final long DATA_LENGTH_WARNING = 128 * 1024 * 1024;

	/** Invalid */
	public static final int TYPE_INVALID = 0; /* This one must be 0 */
	/** Java 8 bit signed */
	public static final int TYPE_BYTE = 1;
	/** Java 16 bit signed */
	public static final int TYPE_SHORT = 2;
	/** Java 32 bit signed */
	public static final int TYPE_INT = 3;
	/** Java 64 bit signed */
	public static final int TYPE_LONG = 4;
	/** Java 32 bit signed floating point */
	public static final int TYPE_FLOAT = 5;
	/** Java 64 bit signed floating point */
	public static final int TYPE_DOUBLE = 6;
	/** Java strings */
	public static final int TYPE_STRING = 7;
	
	protected boolean enableCompression = true;

	/** Conversion from DataMatrix type to HDF5 Type Class */
	private final int toHDF5Class[] = { -1, // TYPE_INVALID
			HDF5Constants.H5T_INTEGER, // TYPE_BYTE
			HDF5Constants.H5T_INTEGER, // TYPE_SHORT
			HDF5Constants.H5T_INTEGER, // TYPE_INT
			HDF5Constants.H5T_INTEGER, // TYPE_LONG
			HDF5Constants.H5T_FLOAT, // TYPE_FLOAT
			HDF5Constants.H5T_FLOAT, // TYPE_DOUBLE
			-1, // TYPE_STRING - Must be handled separately
	};

	/** Conversion from DataMatrix type to HDF5 Type Size */
	private final int toHDF5Size[] = { -1, // Invalid
			1, 2, 4, 8, // Integers
			4, 8, // Floating points
			-1, // Strings handled separately
	};

	/** Conversion from DataMatrix type to HDF5 Native Type ID */
	private final int toHDF5Native[] = { -1, // TYPE_INVALID
			HDF5Constants.H5T_NATIVE_INT8, // TYPE_BYTE
			HDF5Constants.H5T_NATIVE_INT16, // TYPE_SHORT
			HDF5Constants.H5T_NATIVE_INT32, // TYPE_INT
			HDF5Constants.H5T_NATIVE_INT64, // TYPE_LONG
			HDF5Constants.H5T_NATIVE_FLOAT, // TYPE_FLOAT
			HDF5Constants.H5T_NATIVE_DOUBLE, // TYPE_DOUBLE
			-1, };

	/** Conversion from DataMatrix type to Java class object */
	private final Class<?> toJavaClass[] = { null, // TYPE_INVALID
			byte.class, // TYPE_BYTE
			short.class, // TYPE_SHORT
			int.class, // TYPE_INT
			long.class, // TYPE_LONG
			float.class, // TYPE_FLOAT
			double.class, // TYPE_DOUBLE
			String.class, // TYPE_STRING
	};
	
	/** Conversion from DataMatrix type to String description */
		private final String toString[] = { "Invalid/Untyped", // TYPE_INVALID
			"Byte(8-bit)", // TYPE_BYTE
			"Short(16-bit)", // TYPE_SHORT
			"Int(32-bit)", // TYPE_INT
			"Long(64-bit)", // TYPE_LONG
			"Float(32-bit)", // TYPE_FLOAT
			"Double(64-bit)", // TYPE_DOUBLE
			"String", // TYPE_STRING
	};

	/** rank of data held */
	private int rank;

	/** Array dimensions, dims[0] is shallowest dims[rank-1] is deepest. */
	private long dims[];

	/** DataMatrix TYPE_ ?? ID for data */
	private int dataType = TYPE_INVALID;
	private int coordTypes[];

	/** The Data */
	private Object data;

	/** The coordinate vectors */
	private Object coords[];

	/** Create an uninitialised DataMatrix */
	public DataMatrix() {

	}
	
	public String toString(){
		String str = rank + "D Data Matrix of type " + toString[dataType];
		if(data == null) str += ", Null Data";
		if(dims != null){
			str += " (" + dims[0];
			for(int i=1; i < rank; i++)
				str += "x" + dims[i];
			str += ")";
		}else str += ", No dims";
		if(coordTypes != null && coords != null){
			str += ", coords:";
			for(int i=0; i<rank; i++)
				if(coords[i] != null)
					str += " "+i+"="+toString[coordTypes[i]]+"("+Array.getLength(coords[i])+")";
		}else
			str += ", Null coords/coordTypes";
		return str;
	}

	/** Create empty DataMatrix of given rank */
	public DataMatrix(int rank) {
		this.rank = rank;
		this.dims = new long[rank];
		this.coords = new Object[rank];
		this.coordTypes = new int[rank]; // All init to invalid
	}
	
	public DataMatrix(Object data, Object coords[]){ 
		initFromDataAndCoords(data, coords);
	}

	/** Create DataMatrix object based on nD array and coordinate set */
	public void initFromDataAndCoords(Object data, Object coords[]) {
		
		this.data = data;
		int ret[] = getObjectRankAndType(data);
		this.rank = ret[0];
		this.dataType = ret[1];

		if (coords.length != rank)
			throw new IllegalArgumentException(
					"Expecting coords[] to have same number of elements as rank of data");
		this.coords = new Object[rank];
		this.coordTypes = new int[rank]; // All init to invalid/empty
		this.dims = new long[rank];

		getObjectDims(data, dims);

		for (int i = 0; i < rank; i++)
			setCoords(i, coords[i]);
	}

	/** Create DataMatrix object based on nD array */
	public DataMatrix(Object data) {
		initFromData(data);
	}
	
	/** Init DataMatrix object based on nD array. This will erase everything including coords. */
	public void initFromData(Object data){
		int ret[] = getObjectRankAndType(data);
		this.data = data;
		this.rank = ret[0];
		this.dataType = ret[1];
		this.coords = new Object[rank];
		this.coordTypes = new int[rank]; // All init to invalid/empty
		this.dims = new long[rank];
		getObjectDims(data, dims);

	}

	/** Create DataMatrix from HDF5 file */
	public DataMatrix(String fileName) throws IOException {
		loadFromHD5(fileName);
	}
	
	/** Returns the rank of the data held */
	public final int getRank() {
		return rank;
	}

	/** Get data dimensions, dims[0] is shallowest dims[rank-1] is deepest. */
	public final long[] getDims() {
		return dims;
	}

	/** Returns array of objects, each giving coordinates in each dimension */
	public final Object[] getAllCoords() {
		return coords;
	}

	/** Returns array of coordinates */
	public final Object getCoords(int dim) {
		return coords[dim];
	}

	/** Returns multidimensional data object */
	public final Object getData() {
		return data;
	}
	
	/** Returns the data converted to specified type */
	public final Object getDataAsType(Class<?> type){
		int iDims[] = new int[rank];
		for(int i=0;i<rank;i++)
			iDims[i] = (int)dims[i];
		Object dest = java.lang.reflect.Array.newInstance(type, iDims);
		DataConvertPureJava.convertMultiDimArray(data, dest);
		return dest;		
	}
	
	/** Returns the coordinates for specified dimension converted to specified type */
	public final Object getCoordsAsType(int dim, Class<?> type){
		
		Object dest = java.lang.reflect.Array.newInstance(type, (int)dims[dim]);
		DataConvertPureJava.convert1DArray(coords[dim], dest);
		
		return dest;		
	}

	/**
	 * Set coordinates for dimension dim (coords[0] is for shallowest dimension
	 * of data[]...[])
	 */
	public final void setCoords(int dim, Object coords) {
		// Figure out type and set it, also set dim length when we can cast
		long coordLen = 0;

		if (coords == null) {
			coordTypes[dim] = TYPE_INVALID; // no coords so no type
			return;
			// can't set the dims
		}else if (coords instanceof byte[]) coordTypes[dim] = TYPE_BYTE;
		else if (coords instanceof int[]) coordTypes[dim] = TYPE_INT;
		else if (coords instanceof short[]) coordTypes[dim] = TYPE_SHORT;
		else if (coords instanceof long[]) coordTypes[dim] = TYPE_LONG;
		else if (coords instanceof float[]) coordTypes[dim] = TYPE_FLOAT;
		else if (coords instanceof double[]) coordTypes[dim] = TYPE_DOUBLE;
		else if (coords instanceof String[]) coordTypes[dim] = TYPE_STRING;
		else throw new IllegalArgumentException("Unsupported type");
		
		coordLen = Array.getLength(coords);

		if (coordLen != dims[dim])
			throw new IllegalArgumentException("Length of coords array ("
					+ coordLen + ")for dimension " + dim
					+ " does not match dimension size " + dims[dim]);

		this.coords[dim] = coords;
	}

	/**
	 * Finds rank and type of nD array
	 * 
	 * @return int[]{ rank, type } where type is one of DataMatrix.TYPE_*
	 */
	public static final int[] getObjectRankAndType(Object obj) {
		int rank = 1, type = TYPE_INVALID;
		Object nextLevel;
		nextLevel = obj;

		/* Keep going deeper until it's not an array anymore */
		while (nextLevel instanceof Object[]) {
			nextLevel = java.lang.reflect.Array.get(nextLevel, 0);
			rank++;
		}
		;

		if (nextLevel instanceof String) {
			type = TYPE_STRING;
			// The string will have been picked up an iteration too late
			// Since (string[] == object[]) whereas (int[] != object[])
			rank--;
		}else if (nextLevel instanceof byte[])
			type = TYPE_BYTE;
		else if (nextLevel instanceof short[])
			type = TYPE_SHORT;
		else if (nextLevel instanceof int[])
			type = TYPE_INT;
		else if (nextLevel instanceof long[])
			type = TYPE_LONG;
		else if (nextLevel instanceof float[])
			type = TYPE_FLOAT;
		else if (nextLevel instanceof double[])
			type = TYPE_DOUBLE;
		else
			throw new IllegalArgumentException("Unsupported type");

		return new int[] { rank, type };
	}

	/**
	 * Fill in dimensions array of object obj (which must have rank of the
	 * length of dims)
	 * 
	 * @return The total number of entries in the array (product of dims)
	 */
	public static final long getObjectDims(Object obj, long dims[]) {
		Object nextLevel = obj;
		int rank = dims.length;
		long nEntries = 1;

		/*
		 * Go down recasting, getting the dimension size each time. We ASSUME
		 * each dimension is a constant size for all positions in the other
		 * dimensions, otherwise this would have to be recursive
		 */
		for (int i = 0; i < rank; i++) {
			dims[i] = java.lang.reflect.Array.getLength(nextLevel);
			nextLevel = java.lang.reflect.Array.get(nextLevel, 0);
			nEntries *= dims[i];
		}

		return nEntries;
	}

	
	/** Override to read extra information when reading HDF5 files */
	protected void readExtraFromHDF5(int file_id) throws HDF5LibraryException,
			IOException, HDF5Exception { /* nothing by default */
	}

	/** Override to write extra information when writing HDF5 files */
	protected void writeExtraToHDF5(int file_id) throws HDF5LibraryException,
			IOException, HDF5Exception { /* nothing by default */
	}
	

	/** Check all directories required to write 'fileName' exist */
	private static void makePath(String fileName){
		//build dir up to and not including file name
		String bits[] = fileName.split("\\/");
		if(bits.length <= 1)return; //file into current dir, ignore
		
		//recompile the directory 
		StringBuffer dir = new StringBuffer("");
		for(int i=0;i<bits.length-1;i++){
			dir.append("/");
			dir.append(bits[i]);
		}
		(new File(dir.toString())).mkdirs();
	}
		
	/** Writes object to HDF5 File */
	public final void writeToHD5(String fileName) throws IOException {
		fileName = fileNameFixup(fileName);
		makePath(fileName);
		try {
			(new File(fileName)).delete();
			int file_id = H5.H5Fcreate(fileName, HDF5Constants.H5F_ACC_TRUNC,
					HDF5Constants.H5P_DEFAULT, HDF5Constants.H5P_DEFAULT);

			// write the actual data
			writeHD5DataSpace(file_id, "/data", this.rank, this.dims,
					this.data, dataType);

			// write each coordinates dimensions
			for (int i = 0; i < rank; i++)
				if (coords[i] != null)
					writeHD5DataSpace(file_id, "/coord" + i, 1,
							new long[] { dims[i] }, this.coords[i],
							coordTypes[i]);

			// Allow anyone derived from us to write extra info
			writeExtraToHDF5(file_id);

			H5.H5Fclose(file_id);

		} catch (HDF5Exception e) {
			throw new IOException(e); // Don't want to deal with this here
											// and avoid catches
		}
	}

	/** Write an array of given type to dataspace in a HDF5 file */
	protected final void writeHD5DataSpace(int file_id, String name, int rank,
			long dims[], Object data, int dataType) throws IOException {
		int type_id, dataspace_id = -1, dataset_id = -1;
		Object dataToWrite;

		if (data == null)
			return; // Nothing todo
		if (dataType == TYPE_INVALID)
			throw new IllegalArgumentException("No data type for non-null data");
	
		try {
			/*
			 * Arrays of strings require some special handling since the JHI
			 * only supports writing of fixed length strings.
			 */
			if (dataType == TYPE_STRING) {

				// Get biggest string length and then calc total bytes required
				int maxLen = DataConvertPureJava.getMaxStringLength(data);
				long totalBytes = maxLen;

				for (int i = 0; i < rank; i++)
					totalBytes *= dims[i];

				// Check that we haven't been asked to do something completely
				// insane
				if (totalBytes > DATA_LENGTH_WARNING)
					System.err
							.println("WARNING: DataMatrix is attempting to build "
									+ totalBytes + " bytes of string data!!");

				// Build the buffer
				// ** CHECK INTEGER OVERFLOW IF ERROR HERE **
				byte strData[] = new byte[(int) totalBytes];
				int bytesDone = DataConvertPureJava.flattenAndNullPadStrings(strData, 0, data,
						maxLen);
				if(bytesDone != totalBytes)			
					throw new RuntimeException("Byte count mismatch when flattening strings. Total size should be " +
								totalBytes + " but " + bytesDone + " bytes were encoded.");
								
				// Build a special type_id for the fixed length string
				type_id = H5.H5Tcopy(HDF5Constants.H5T_C_S1);
				H5.H5Tset_size(type_id, maxLen);
				dataToWrite = strData;

			} else {
				// type_id = toHDF5Type[dataType]; //Convert data type to HDF5
				// Type ID
				// type_id = H5.H5Tcreate(toHDF5Class[dataType],
				// toHDF5Size[dataType]);
				type_id = toHDF5Native[dataType]; // Convert data type to native
													// HDF5 Type ID
				dataToWrite = data; // Just write the data
			}

			dataspace_id = H5.H5Screate_simple(rank, dims, null);
			
			int plist_id;
			if(enableCompression){
				plist_id = H5.H5Pcreate(HDF5Constants.H5P_DATASET_CREATE);
				H5.H5Pset_chunk(plist_id, rank, dims);
				H5.H5Pset_deflate(plist_id, 6);
			}else
				plist_id = HDF5Constants.H5P_DEFAULT;

			dataset_id = H5.H5Dcreate(file_id, name, type_id, dataspace_id,
					plist_id);

			try {
				H5.H5Dwrite(dataset_id, type_id, HDF5Constants.H5S_ALL,
						HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT,
						dataToWrite);

			} catch (HDF5Exception e) {
				// Don't want to deal with this here and avoid catches
				throw new IOException(e);
			}

		
		} catch (HDF5Exception e1) {
			// Don't want to deal with this here and avoid catches
			throw new IOException(e1);
		} finally {
			try {
				H5.H5Dclose(dataset_id);
				H5.H5Sclose(dataspace_id);
			} catch (HDF5LibraryException e) {
				System.err.println("WARNING: HDF5 Dataspace/Dataset close failed");
			}
			
		}
	}

	/**
	 * read a dataspace and return rank, dims and the data
	 * 
	 * @return Object[]{ int dataType, long dims[], Object matrix[][] }
	 * @throws HDF5Exception
	 * @throws NullPointerException
	 * @throws HDF5LibraryException
	 * */

	protected final Object[] readHD5DataSpace(int file_id, String name)
			throws IOException, HDF5Exception {

		int rank;
		Integer dataType = TYPE_INVALID;
		long dims[];
		Object data;
		 
		int dataset_id = -1, dataspace_id = -1;
		
		try{
			// open the data set
			
			dataset_id = H5.H5Dopen(file_id, name);
	
	
			// Get dataspace rank and dims
			dataspace_id = H5.H5Dget_space(dataset_id);
			rank = H5.H5Sget_simple_extent_ndims(dataspace_id);
			dims = new long[rank];
			H5.H5Sget_simple_extent_dims(dataspace_id, dims, null);
	
			// get data type_id, class and size
			int type_id = H5.H5Dget_type(dataset_id);
			int type_class = H5.H5Tget_class(type_id);
			int type_size = H5.H5Tget_size(type_id);
	
			if (type_class == HDF5Constants.H5T_STRING) {
	
				long totalBytes = type_size;
				
				for (int i = 0; i < rank; i++)
					totalBytes *= dims[i];
	
				
				if (totalBytes > DATA_LENGTH_WARNING)
					System.err.println("WARNING: DataMatrix is attempting to build "
									+ totalBytes + " bytes of string data!!");
	
				// Allocate the string buffer
				// ** CHECK INTEGER OVERFLOW IF ERROR HERE **
				byte strData[] = new byte[(int) totalBytes];
	
				H5.H5Dread(dataset_id, type_id, HDF5Constants.H5S_ALL,
						HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT, strData);
	
				// Create the array of strings
				int iDims[] = new int[rank]; // inverted integer dimensions
				for (int i = 0; i < rank; i++)
					iDims[i] = (int) dims[i];
				data = java.lang.reflect.Array.newInstance(String.class, iDims);
	
				int bytesDone = DataConvertPureJava.unflattenStrings(strData, 0, data, type_size);
				
				if(bytesDone != totalBytes)			
					throw new RuntimeException("Byte count mismatch when unflattening strings. Total size was " +
								totalBytes + " but " + bytesDone + " bytes were extracted.");
				
				dataType = TYPE_STRING;
			} else { // Not string
	
				/* Find a match for both the class and size */
				for (dataType = 0; dataType < toHDF5Class.length; dataType++)
					if (type_class == toHDF5Class[dataType]
							&& type_size == toHDF5Size[dataType])
						break;
	
				if (dataType >= toHDF5Class.length) {
					dataType = TYPE_INVALID;
					throw new RuntimeException("Unknown data type " + type_id
							+ " in HDF5 file.");
				}
	
				// Create the data object
				int iDims[] = new int[rank]; // inverted integer dimensions
				for (int i = 0; i < rank; i++)
					iDims[i] = (int) dims[i];
				data = java.lang.reflect.Array.newInstance(toJavaClass[dataType],
						iDims);
	
				H5.H5Dread(dataset_id, type_id, HDF5Constants.H5S_ALL,
						HDF5Constants.H5S_ALL, HDF5Constants.H5P_DEFAULT, data);
	
			}		
		} finally {
			try{
				H5.H5Dclose(dataset_id);
				H5.H5Sclose(dataspace_id);
			}catch( HDF5Exception e){
				System.err.println("WARNING: HDF5 dataspace/dataset close failed");
				
			}
		}

		return new Object[] { dataType, dims, data };
	}
	
	/** Get 0D String from HDF5 file and checks it is 0D. Probably only used by readExtraFromHDF5() in derived classes */ 
	protected String readHDF5SingleString(int file_id, String name) throws IOException, HDF5Exception {
			Object ret[] = readHD5DataSpace(file_id, name);
			if( ret[2] instanceof String)
				return (String)ret[2];
			else if( ret[2] instanceof String[] && ((String[])ret[2]).length == 1)
				return ((String[])ret[2])[0];
			else
				throw new RuntimeException("HDF5 dataset '"+name+"' isn't a String or String[1]");

	}
	
	/** Get 0D int from HDF5 file and checks it is 0D. Returns 0 if it doesn't exist.
	 * Probably only used by readExtraFromHDF5() in derived classes. */ 
	protected int readHDF5SingleInt(int file_id, String name) throws IOException, HDF5Exception {		
		Object ret[] = readHD5DataSpace(file_id, name);
		if( ret[2] instanceof Integer) 
			return (int)(Integer)ret[2];
		else if( ret[2] instanceof int[] && ((int[])ret[2]).length == 1)
			return ((int[])ret[2])[0];
		else
			throw new RuntimeException("HDF5 dataset '"+name+"' isn't int or int[1]");						
	}
	
	/** removes multiple slashes from a path */
	private static String fileNameFixup(String in){
		String tmp;
		in = in.replaceAll("\\\\", "/");
		do{			
			tmp = in;
			in = tmp.replaceAll("//", "/");			
		}while(in.length() < tmp.length());
		
		in = in.replaceAll("\\<", "_lt_");
		in = in.replaceAll("\\>", "_gt_");
		
		return in;
	}	

	/** Loads object from HDF5 File */
	public final void loadFromHD5(String fileName) throws IOException {
		this.data = null;
		this.dims = null;
		this.coords = null;
		this.coordTypes = null;
		this.rank = -1;
		this.dataType = TYPE_INVALID;

		fileName = fileNameFixup(fileName);
		if(!(new File(fileName)).canRead())
			throw new FileNotFoundException("Precheck failed: Cannot read HDF5 file '"+fileName+"'.");
		
		int file_id;
		try {
			file_id = H5.H5Fopen(fileName, HDF5Constants.H5F_ACC_RDONLY,
					HDF5Constants.H5P_DEFAULT);

			Object ret[];

			try {
				ret = readHD5DataSpace(file_id, "/data");
			} catch (HDF5SymbolTableException err) {
				throw new IllegalArgumentException("'data' entry in HDF5 file "
						+ fileName + " not found.");
			}

			this.dataType = ((Integer) ret[0]);
			this.dims = (long[]) ret[1];
			this.rank = dims.length;
			this.data = ret[2];

			this.coordTypes = new int[rank];
			this.coords = new Object[rank];

			for (int dim = 0; dim < rank; dim++) {
				try {
					ret = readHD5DataSpace(file_id, "/coord" + dim);

					this.coordTypes[dim] = ((Integer) ret[0]);
					long coordDims[] = (long[]) ret[1];
					if (coordDims.length != 1)
						throw new IllegalArgumentException(
								"Coordinates array for dimension " + dim
										+ " in HDF5 file " + fileName
										+ " doesn't have rank 1");
					this.coords[dim] = ret[2];
				} catch (HDF5SymbolTableException err) {
					this.coordTypes[dim] = TYPE_INVALID;
					this.coords[dim] = null;
				}
			}

			// Allow those who inherit us to get extra info from the file
			readExtraFromHDF5(file_id);

		} catch (HDF5Exception e) {
			throw new IOException(e);
		}
		
		try{
			H5.H5Fclose(file_id);
		}catch(HDF5Exception e){
			System.err.println("WARNING: HDF5 Close failed.");
		}
	}

	public final boolean isEnableCompression() { return enableCompression; }
	public final void enableCompression() {	this.enableCompression = true;	}
	public final void disableCompression() { this.enableCompression = false;	}

}
