test update

CMrcReader.cpp → CMrcReader.cu

@@ -6,11 +6,7 @@
  */
 
 #include <CMrcReader.h>
-
-
-CMrcReader::CMrcReader(std::string mrc_file_name) : // @suppress("Class members should be properly initialized")
-		mrc_file_name(mrc_file_name) { // @suppress("Class members should be properly initialized")
-}
+CMrcReader::CMrcReader(){}
 
 void CMrcReader::printHeader() {
 	if (!x_HeaderRead) {
@@ -266,7 +262,7 @@ void CMrcReader::readHeader() {
 	x_FileHandle.close();
 }
 
-void CMrcReader::readData() {
+int CMrcReader::readData() {
 	//    * mode 0 -> int8
 	//    * mode 1 -> int16
 	//    * mode 2 -> float32
@@ -287,8 +283,9 @@ void CMrcReader::readData() {
 		//jump header
 		x_FileHandle.seekg(1024, std::ios::beg);
 		x_FileHandle.read((char*) data, bytes);
-
+		return 0;
 	}
+	return -1;
 }
 
 CMrcReader::~CMrcReader() {

CMrcReader.h

@@ -153,15 +153,15 @@ typedef unsigned char uchar;
 class CMrcReader {
 public:
 	CMrcReader();
-	CMrcReader(std::string mrc_file_name);
 	SMrcHeader header;
-	std::string mrc_file_name;
-	float * data;
+	const char * mrc_file_name = nullptr;
+	float * data = nullptr;
 	virtual ~CMrcReader();
+	void setFileName(const char * file_name){mrc_file_name = file_name;}
 	void readHeader();
 	void write();
 	void printHeader();
-	void readData(void);
+	int readData(void);
 	void setDimensions(float dim[3],uint pixel_dim[3]);
 	void makeCube(float dim[3],uint pixel_dim[3],float pos[3],float cube_dim[3],
 			float cube_val, float non_cube_val);

Density.h

@@ -35,12 +35,12 @@
 #include <chrono>
 #include <nvml.h>
 
+#include <MemoryResources.h>
+
 inline size_t vol(uint4 vec){
 	return vec.x*vec.y*vec.z;
 }
 
-
-
 class Density {
 public:
 	Density();
@@ -61,10 +61,8 @@ public:
 	uint * d_pixel_dim;
 	uint * h_padding_dim;
 	uint * d_padding_dim;
-	thrust::host_vector<float> th_data;
-	thrust::device_vector<float> td_data;
-	float * h_data;
-	float * d_data;
+	TDensity * h_data;
+	TDensity * d_data;
 	float * no_padding;
 	float * h_pixel_size;
 	float * d_pixel_size;
@@ -98,7 +96,7 @@ public:
 	__host__ int add_z_plane(int z);
 	__host__ int * d_above_threshold(thrust::device_vector<int> & above,float t);
 	__host__ int * h_above_threshold(thrust::host_vector<int> & above,float t);
-	__host__ void fill_with(float,bool copy_to_device = true);
+	__host__ void fill_with(TDensity,bool copy_to_device = true);
 	__host__ void empty_copy(Density &density,float fill);
 	__host__ void init_device(void);
 	__host__ void to_device(void);
@@ -118,7 +116,6 @@ public:
 	__host__ float h_max_dim(void);
 	__device__ float d_max_dim(void);
 	__host__ void indices_in_range(thrust::device_vector<uint> & td_indices, bool inside, const float & rho0, const float & rho1, size_t & size);
-	void register_mrc_source(CMrcReader * reader){mrc_sources.push_back(reader);};
 	__host__ uint4 h_pixel_index_to_pixel(const uint & pixel_index);
 	__device__ uint4 d_pixel_index_to_pixel(const uint & pixel_index);
 	__host__ float4 h_pixel_linear_index_to_coord(const uint & pixel_index);
@@ -128,7 +125,7 @@ public:
 	__device__ bool contains_pixel(uint4 & index);
 	__device__ uint4 calc_pixel_indices(size_t pixel_index);
 private:
-	std::vector<CMrcReader *> mrc_sources;
+	CMrcReader mrc_source;
 
 };
 

FittingProtocol.cu

@@ -6,12 +6,43 @@
  */
 #include <FittingProtocol.h>
 
+void FittingProtocol::_determine_gpu_resources(void){
+	//memory management library
+	nvmlInit_v2();
+	cudaGetDeviceCount(&_num_devices);
+	_device_handles.resize(_num_devices);
+	_device_memory_infos.resize(_num_devices);
+	for (int i=0;i<_num_devices;i++){
+		nvmlDeviceGetHandleByIndex_v2(i,&_device_handles[i]);
+		nvmlDeviceGetMemoryInfo(_device_handles[i], &_device_memory_infos[i]);
+	}
+	CudaCheckError();
+}
+
 FittingProtocol::FittingProtocol() {
-	// TODO Auto-generated constructor stub
+	_determine_gpu_resources();
+}
 
+void FittingProtocol::printGPUInfo(void){
+	printf("Found %i devices ...\n",_num_devices);
+	printf("-------------------------------------");
+	for (int i=0;i<_num_devices;i++){
+		printf("Device %i:\n",i);
+		printf("Total Memory: %u bytes\n",_device_memory_infos[i].total);
+		printf("-------------------------------------");
+	}
 }
 
+
 FittingProtocol::~FittingProtocol() {
 	// TODO Auto-generated destructor stub
 }
 
+void FittingProtocol::setFileTarget(const char * file_name){
+	if (file_exists(file_name)){
+//		target = Target::fromMrc(file_name);
+	}
+	else {
+//		std::cout << "File " << file_name << " is not found ..." << std::endl;
+	}
+}

FittingProtocol.h

@@ -8,19 +8,33 @@
 #ifndef FITTINGPROTOCOL_H_
 #define FITTINGPROTOCOL_H_
 
-#include <Target.h>
-#include <Query.h>
-#include <Score.h>
+//#include <Target.h>
+//#include <Query.h>
+//#include <Score.h>
 #include <vector>
 #include <MemoryResources.h>
+#include <util.h>
+#include <nvml.h>
+#include <vector>
 
 class FittingProtocol {
 public:
 	FittingProtocol();
+	void setFileTarget(const char * file_name);
 	virtual ~FittingProtocol();
-	Target target;
-	std::vector<Query> queries;
-	std::vector<Score> scores;
+	void printGPUInfo(void);
+//	Target target;
+//	std::vector<Query> queries;
+//	std::vector<Score> scores;
+private:
+	size_t _byte_vol = 0;
+	size_t _bytes_used = 0;
+	std::vector<nvmlDevice_t> _device_handles;
+	std::vector<nvmlMemory_t> _device_memory_infos;
+	int _num_devices = 0;
+	float _gpu_mem_factor = 0.8;
+	void * _gpu_space = nullptr;
+	void _determine_gpu_resources(void);
 };
 
 #endif /* FITTINGPROTOCOL_H_ */

Makefile.test

@@ -14,6 +14,10 @@ LFLAGS=-L. $(CUDA_LINKS) $(BOOST_LINKS)
 LINK_LORM = -L/data/tools/lorm/build/lib -llorm -lLORM_NFFTwrapper_obj
 LINK_NFFT3 = -L/usr/local/lib -lnfft3
 
+ifndef DEVICE_ARCH
+	DEVICE_ARCH = sm_61
+endif
+
 
 eval: GpuDelaunay.o KerDivision.o Splaying.o Star.o PredWrapper.o RandGen.o InputCreator.o predicates.o ThrustWrapper.o KerPredicates.o CRotationGrid.o DelaunayChecker.o
 	$(NVCC) --gpu-architecture=sm_61 --include-path=./,./gDel3D/GPU/ CRotationGrid.o InputCreator.o RandGen.o DelaunayChecker.o KerPredicates.o Splaying.o Star.o predicates.o ThrustWrapper.o PredWrapper.o KerDivision.o GpuDelaunay.o TransformationGrid.cu -o eval
@@ -21,9 +25,20 @@ eval: GpuDelaunay.o KerDivision.o Splaying.o Star.o PredWrapper.o RandGen.o Inpu
 ccl: CMrcReader.o
 #	$(NVCC) --gpu-architecture=sm_61 --include-path=./ $(CUB_INCLUDE) --device-c Labeler.cu Density.cu Kernels.cu Particles.cu TransformationGrid.cu CRotationGrid.cu ccl_test.cu -g -G
 	$(NVCC) --gpu-architecture=sm_61 --include-path=./ $(CUB_INCLUDE) --device-c Density.cu TransformationGrid.cu ccl_test.cu -g -G
-	$(NVCC) --gpu-architecture=sm_61 --device-link Density.o ccl_test.o Kernels.o Labeler.o Particles.o PdbReader.o TransformationGrid.o CRotationGrid.o --output-file link.o
+	$(NVCC) --gpu-architecture=sm_61 --device-link Density.o ccl_test.o Kernels.o Labeler.o Particles.o PdbReader.o TransformationGrid.o CRotationGrid.o --output-file link.o $(CUDA_LINKS)
 	g++ Density.o Particles.o PdbReader.o CMrcReader.o Kernels.o TransformationGrid.o Labeler.o CRotationGrid.o ccl_test.o link.o -o ccl_test -L. $(CUDA_LINKS) $(BOOST_LINKS) -lpng -lnvidia-ml
 
+protocol:
+#	sm_72 volta support, V100s
+#	$(NVCC) --gpu-architecture=sm_61 --include-path=./ $(CUB_INCLUDE) --device-c Particles.cu PdbReader.cu CMrcReader.cu Target.cu FittingProtocol.cu Density.cu ccl_test.cu -g -G
+#	$(NVCC) --gpu-architecture=sm_61 --device-link Particles.o PdbReader.o FittingProtocol.o Density.o Target.o CMrcReader.o --output-file link.o -lnvidia-ml $(CUDA_LINKS)
+#	g++ FittingProtocol.o Particles.o PdbReader.o Target.o Density.o CMrcReader.o ccl_test.o link.o -o protocol -lnvidia-ml -L. $(CUDA_LINKS) 
+	$(info Compiling for device architecture $(DEVICE_ARCH))
+	$(NVCC) --gpu-architecture=$(DEVICE_ARCH) --include-path=./ --device-c FittingProtocol.cu ccl_test.cu -g -G
+	$(NVCC) --gpu-architecture=$(DEVICE_ARCH) --device-link FittingProtocol.o ccl_test.o --output-file link.o
+	g++ FittingProtocol.o ccl_test.o link.o -o protocol -lnvidia-ml -L/usr/local/cuda-10.0/lib64/ -lcudart 
+
+
 ccl2: CMrcReader.o 
 	$(NVCC) --gpu-architecture=sm_61 --include-path=./ --device-c Density.cu Kernels.cu PdbReader.cu Particles.cu Labeler.cu ccl_test.cu -g -G
 	$(NVCC) --gpu-architecture=sm_61 --device-link Sampler.o PdbReader.o Particles.o Labeler.o Kernels.o Density.o ccl_test.o --output-file link.o

MemoryResources.h

@@ -9,10 +9,14 @@
 #define MEMORYRESOURCES_H_
 
 
+
+
 struct MemoryResources{
 	size_t GPU;
 	size_t CPU;
 };
 
+enum SourceType {MRC,PDB};
+typedef float TDensity;
 
 #endif /* MEMORYRESOURCES_H_ */

Query.h

@@ -15,10 +15,16 @@
 class Query {
 public:
 	Query();
-	virtual ~Query();
 	static Query fromPdb(const char * fileName);
-	Density * as_density();
-	Particles * as_particles();
+	size_t getCpuMem();
+	size_t getGpuMem();
+	int initCpu();
+	int initGpu();
+	int toGpu();
+	int fromGpu();
+	virtual ~Query();
+//	Density * as_density();
+//	Particles * as_particles();
 };
 
 #endif /* QUERY_H_ */

Score.h

@@ -23,7 +23,6 @@ public:
 	Target * const& target = nullptr;
 	Query * const query = nullptr;
 	virtual MemoryResources requestTargetMemoryResources() = 0;
-	virtual MemoryResources requestTargetMemoryResources() = 0;
 };
 
 #endif /* SCORE_H_ */

Target.cu

@@ -11,8 +11,9 @@ Target::Target() {}
 
 Target Target::fromMrc(const char * fileName){
 	Target target;
-	_density = Density::from_mrc(fileName);
-	_data_byte_vol = _density.h_pixel_vol()*sizeof(float);
+	target._density = Density::from_mrc(fileName);
+	target._data_byte_vol = target._density.h_pixel_vol()*sizeof(*target._density.h_data);
+	target._source_type = MRC;
 	return target;
 }
 
@@ -20,6 +21,40 @@ size_t Target::getCpuMem(){
 	return _data_byte_vol;
 }
 
+int Target::initCpu(){
+	int ret = 0;
+	if (_source_type == MRC){
+		_density.h_data = (TDensity *) malloc(_data_byte_vol);
+		_cpu_initiated = true;
+	}
+	return ret;
+}
+
+int Target::initGpu(){
+	int ret = 0;
+	if (_source_type == MRC){
+		ret = int(cudaSuccess != cudaMalloc((void **)&_density.d_data,_data_byte_vol));
+		_gpu_initiated = true;
+		return ret;
+	}
+	return 0;
+}
+
+int Target::toGpu(){
+	if (_source_type == MRC && _cpu_initiated && _gpu_initiated){
+			return int(cudaSuccess != cudaMemcpy(_density.d_data,_density.d_data,_data_byte_vol,cudaMemcpyHostToDevice));
+	}
+	return 0;
+}
+
+int Target::fromGpu(){
+	if (_source_type == MRC && _cpu_initiated && _gpu_initiated){
+			return int(cudaSuccess != cudaMemcpy(_density.d_data,_density.d_data,_data_byte_vol,cudaMemcpyDeviceToHost));
+	}
+	return 0;
+}
+
+
 size_t Target::getGpuMem(){
 	return _data_byte_vol;
 }

Target.h

@@ -9,6 +9,8 @@
 #define TARGET_H_
 
 #include <Density.h>
+#include <MemoryResources.h>
+#include <util.h>
 
 class Target {
 public:
@@ -16,10 +18,17 @@ public:
 	static Target fromMrc(const char * fileName);
 	size_t getCpuMem();
 	size_t getGpuMem();
+	int initCpu();
+	int initGpu();
+	int toGpu();
+	int fromGpu();
 	virtual ~Target();
 private:
-	Density _density = 0;
+	Density _density;
 	size_t _data_byte_vol = 0;
+	SourceType _source_type;
+	bool _cpu_initiated = false;
+	bool _gpu_initiated = false;
 };
 
 #endif /* TARGET_H_ */

util.h

@@ -10,6 +10,7 @@
 #include <cuda_util_include/cutil_math.h>
 #include <functional>
 #include <thrust/functional.h>
+#include <unistd.h>
 
 //cuda helper function stolen from the interwebs
 #define CUDA_ERROR_CHECK
@@ -46,6 +47,16 @@ inline void __cudaCheckError(const char *file, const int line) {
 	return;
 }
 
+inline bool file_exists (const std::string& name) {
+    if (FILE *file = fopen(name.c_str(), "r")) {
+        fclose(file);
+        return true;
+    } else {
+        return false;
+    }
+}
+
+
 const size_t one_gb = 1073741824;
 static const int MAXTHREADS = 1024;
 struct compare_float4_x