% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% "excread" is used for merging 'exchange'-formatted oceanographic bottle files, e.g. from CCHDO
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% Use:
%   excread is executed as a script, and requires no arguments.
%   The different files to be read and merged must be located
%   in the current working directory (and nothing else).
%   The input files must read ".csv" anywhere (!) in the filename.
%   Output (3 files) is saved in the current working directory
%   Output files have unique numerical prefixes, so no risk of overwriting previous output.
%
% File format:
%   The first line of a datafile is always ignored.
%   Any following lines beginning with a '#' character are ignored.
%   Next line is assumed to hold comma separated names of variables.
%   Next line is assumed to hold comma separated variables' units.
%   From here on, comma separated values.
%   If present, the 'END_DATA' tag at the end of files (and any fields trailing that) 
%     is removed before merging (***functional but not yet foolproof***).
%
% Please note:
%   For variables in different files to be merged, the variable names must be identical.
%   The variable "SECT_ID", if present, is ignored, and is thus not present in final dataset.
%   -999's are assumed to be missing value indicators and are set to NaN.
%   Quality flags are read and stored as regular variables.
%   Matlab-compatible date/time values are generated.
%   Sequential cruiseIDs are generated (vector 'cruiseID'), in alph. order of filenames.
%   All variables are saved as individual vectors, and also in a big array
%   Resulting dataset is saved in .mat-format, as is a logfile (.txt) and an 
%     EXPOCODE-cruiseID lookuptable (.txt), also containing value totals per variable per cruise
%
% Some user-changeable settings are available in the script. Please have a look.
%
% New in 0.6: fixed bug that stopped reading files when encoutering alphanumerical stationnumbers
%
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% This is version 0.6 (2009-JUN-23).
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% 2009-OCT-24 modified by T.Suzuki@MIRC
% 2009-DEC-29 -- 2013-FEB-07
% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


% Clear and start logfile
clear all; clc
fclose('all');
warning('off','MATLAB:MKDIR:DirectoryExists');
mkdir('output');
diary off
diary('output/templog.txt')
tic
excversion='0.6';
savespreadsheet=0; %%%%%%%%%%% USER SETTING %%%%%%%%%%%%%% Optionally save BIG .csv file with output. 



% Define some stuff.
MERGELIST={
'NITRAT' 'NO2ANO3';
'NITRAT_FLAG_W' 'NO2ANO3_FLAG_W';
'AMMONI' 'NH4';
'AMMONI_FLAG_W' 'NH4_FLAG_W';
'TCARBN' 'TCO2';
'TCARBN_FLAG_W' 'TCO2_FLAG_W';
%'PH_TMP' 'PHTEMP';
'NEONER' 'NEONE';
'O18O16' 'O18_O16';
'O18O16_FLAG_W' 'O18_O16_FLAG_W';
% for pacX
%'PH' 'PH_TS';
'PH' 'PH_TOT';
%'PH_FLAG_W' 'PH_TS_FLAG_W';
'PH_FLAG_W' 'PH_TOT_FLAG_W';
%'PH_TMP' 'PH_TEMP';
'CTDSAL_FLAG_W' 'CTDSAL_FLAG';
%%%
};

TRASHLIST={'CS_137';
'CS_137_FLAG_W';
'CS137ER';
'PLUTO';
'PLUTO_FLAG_W';
'PLUTOER';
'SIG0';
'CHLA';
'CHLA_FLAG_W';
'OXYNIT';
'OXYNIT_FLAG_W';
'ARGON';
'ARGON_FLAG_W';
'METHAN';
'METHAN_FLAG_W';
'AZOTE';
'AZOTE_FLAG_W';
'CHLORA';
'CHLORA_FLAG_W';
'FLUOR';
'FLUOR_FLAG_W';
'DELO18';
'DELO18_FLAG_W';
'BARIUM';
'BARIUM_FLAG_W';
'OSNUM';
'MCHFRM';
'MCHFRM_FLAG_W';
'ORIGSTA';
'UREA';
'UREA_FLAG_W';
'PHAEO';
'PHAEO_FLAG_W';
'FLUORO';
'FLUORO_FLAG_W';
'HAARDT';
'HAARDT_FLAG_W';
'PAR';
'PAR_FLAG_W';
'SPAR';
'SPAR_FLAG_W';
'RA_226';
'RA_226E';
'RA_226_FLAG_W';
'RA_228';
'RA_228E';
'RA_228_FLAG_W';
'RA_8D6';
'RA_8D6E';
'RA_8D6_FLAG_W';
'TDN';
'TDN_FLAG_W';
'LIGNIN';
'LIGNIN_FLAG_W';
'RA_224';
'RA_224E';
'RA_224_FLAG_W';
'TH_228';
'TH_228E';
'TH_228_FLAG_W';
'SOMSAL';
'BACT';
'BACT_FLAG_W';
'CTDNO';
'BNLID';
'BEDFORDNO';
'ID';
'POC';
'POC_FLAG_W';
'PON';
'PON_FLAG_W';
'DOC';
'DOC_FLAG_W';
'DON';
'DON_FLAG_W';
'TOC';
'TOC_FLAG_W';
'INDEX';
'SBE35';
'SBE35_FLAG_W';
'DWNPRS';
'DWNPRS_FLAG_W';
'DWNOXY';
'DWNOXY_FLAG_W';
'SILUNC';
'NRAUNC';
'NRIUNC';
'PHPUNC';
'TON';
'TON_FLAG_W';
'CTDRAW';
'BEDFORT';
'REVPRS';
'REVPRS_FLAG_W';
'REVTMP_FLAG_W';
'REVTMP';
'STHETA';
'PPHYTN';
'TRITER_FLAG_W';
'FCO2';
'FCO2_TMP';
'FCO2_FLAG_W';
'PCO2';
'PCO2_TMP';
'PCO2TMP';
'PCO2_FLAG_W';
'FCO220C';
'FCO220C_FLAG_W';
'FCO2IN';
% for pacX
'PH_NBS';
'PH_NBS_FLAG_W';
'PH_SCALE_FLAG_W';
'AOU';
'AOU_FLAG_W';
'TOTALP';
'TOTALP_FLAG_W';
'PPHYTN';
'PPHYTN_FLAG_W';
'DEPTH_FLAG';
'CTDDEP';
'SIGTHT';
'CTDDPT';
'CTDDPT_FLAG_W';
'CHLWELSH';
'CHLWELSH_W';
'CTDPRS_FLAG_W';
'CTDTMP_FLAG_W';
'DELC14';
'DELC14_FLAG_W';
'C14ERR';
'BTLNBR_FLAG';
'BTLNBR_FLAG_W';
'XMISS';
'XMISS_FLAG_W';
'OPTOXY';
'OPTOXY_FLAG_W';
'FCO2TMP';
'CH4';
'CH4_FLAG_W';
'DTN';
'DTN_FLAG_W';
'CDOM325';
'CDOM325_FLAG_W';
'CDOM340';
'CDOM340_FLAG_W';
'CDOM380';
'CDOM380_FLAG_W';
'CDOM412';
'CDOM412_FLAG_W';
'CDOM443';
'CDOM443_FLAG_W';
'CDOM490';
'CDOM490_FLAG_W';
'CDOM555';
'CDOM555_FLAG_W';
'CDOMSL';
'CDOMSL_FLAG_W';
'CDOMSN';
'CDOMSN_FLAG_W';
'O18_O2';
'O18_O2_FLAG_W';
'CDOM2C';
'CDOM2C_FLAG_W';
'CDOM3C';
'CDOM3C_FLAG_W';
'LAB_DEN';
'LAB_DEN_FLAG_W';
'DELSI30';
'DELSI30_FLAG_W';
'N2';
'N2_FLAG_W';
'N15_N2';
'N15_N2_FLAG_W';
'THETA_FLAG_W';
'PH_TMP_FLAG_W';
'PH_SCALE_FLAG_W';
'PCO2_TMP_FLAG_W';
'DOC_FLAG';
'BTL_LAT';
'BTL_LON';
'REFTMP';
'REFTMP_FLAG_W';
'14C_DOC';
'14C_DOC_FLAG_W';
'PH_TMP';
'PHTEMP';
'PH_TEMP';
%%%
};

OKLIST={
'cruiseID';
'DATE';
'TIME';
'DATETIME';
'STNNBR';
'CASTNO';
'SAMPNO';
'LATITUDE';
'LONGITUDE';
'DEPTH';
'SAMPLEDEPTH';
'CTDPRS';
'CTDPRS_FLAG_W';
'CTDTMP';
'CTDTMP_FLAG_W';
'CTDSAL';
'CTDSAL_FLAG_W';
'BTLNBR_FLAG_W';
'THETA';
'SALNTY';
'SALNTY_FLAG_W';
'CTDOXY';
'CTDOXY_FLAG_W';
'OXYGEN';
'OXYGEN_FLAG_W';
'SILCAT';
'SILCAT_FLAG_W';
'NITRIT';
'NITRIT_FLAG_W';
'PHSPHT';
'PHSPHT_FLAG_W';
'ALKALI';
'ALKALI_FLAG_W';
'PH';
'PH_FLAG_W';
%'PH_TMP';
'PH_SWS';
'PH_SWS_FLAG_W';
'CFC_11';
'CFC_11_FLAG_W';
'CFC_12';
'CFC_12_FLAG_W';
'CFC113';
'CFC113_FLAG_W';
'CCL4';
'CCL4_FLAG_W';
'SF6';
'SF6_FLAG_W';
'TRITUM';
'TRITUM_FLAG_W';
'TRITER';
'HELIUM';
'HELIUM_FLAG_W';
'HELIER';
'DELHE3';
'DELHE3_FLAG_W';
'DELHER';
'DELC14';
'DELC14_FLAG_W';
'C14ERR';
'DELC13';
'DELC13_FLAG_W';
'C13ERR';
'NEON';
'NEON_FLAG_W';
'NEONER_FLAG_W'};



% Search for files
A=dir('*.csv*');
nof=length(A); % number of files
writefrom=1; % Set initial position form which to write data
% If no files found, report error
if nof==0
	disp('*** excread error')
	disp('    No files found in current working directory.')
	disp('    NOTE: files must be of .csv format.')
	disp('*** excread aborted')
	return
end



% Determine variables' preallocation size
lpmb=7000; % Estimated number of lines per MB (used for estimating preallocation size), should be OK for regular CDIAC files
Tbytes=sum([A(:).bytes]');
pal=ceil((Tbytes/1024^2)*lpmb*1.2); % pre-allocation length (adds 20% margin)



% Some feedback to user
disp(cat(2,'*** Number of files found: ',int2str(nof)));
disp(cat(2,'*** Total size of files:   ',int2str(Tbytes/1024),' kB'));
disp(cat(2,'*** Assuming ',int2str(lpmb),' lines per MB, will preallocate ',int2str(pal),'-length variables.'));
disp(' ');



% Preallocate the non-numeric variables and the extra variable cruiseID and TIME (in case not in first datafile)
% the cell array that will hold all the variables encountered in the various datafiles, and add the first one.
EXPOCODE = cell(pal,1);       disp(cat(2,'       Variable "EXPOCODE" preallocated.')); % Class 'cell', so won't be taken up in list 'allvariables', since that will be numerically processed.
cruiseID =  nan(pal,1);       disp(cat(2,'       Variable "cruiseID" preallocated.'));
DATE     =  nan(pal,1);       disp(cat(2,'       Variable "DATE"     preallocated.'));
TIME     = ones(pal,1).*1200; disp(cat(2,'       Variable "TIME"     preallocated.'));
DATETIME =  nan(pal,1);       disp(cat(2,'       Variable "DATETIME" preallocated.')); disp(' ')
% Add four of the extra variables to the list.
allvariables=[{'cruiseID';'DATE';'TIME';'DATETIME'}];
allunits    =[{'';'yyyymmdd';'HHMM';'(for MATLAB)'}];
noalert=0;


% %%%%%%%%%%%%%%%%%%%%%%%%%   THE DATA LOADING BIT  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



% Load and merge the data
for i = 1:1:nof
	
	
		
	% Open the i'th file
	file=A(i).name;
	fid=fopen(file);
	filesize=A(i).bytes;



	% Read line-by-line, until through header (header info is not stored)
	disp(cat(2,'*** Now processing file ',int2str(i),'/',int2str(nof),' : ',file,'.'));
	b=0; stillheader=1;
		while stillheader==1
		B=fgetl(fid);
		b=b+1;		
		if strcmp(B(1),'#')==0 & b>1 % Never finish on first line, which might not have '#' but clearly isn't data
			stillheader=0;
		end		
	end
	
	
	
	% Read variable names
	varstart=[1 strfind(B,',')+1];
	varstop =[strfind(B,',')-1 size(B,2)];
	nov=length(varstart); % Number of variables
	disp(cat(2,'    Number of variables found: ',int2str(nov)));
	variables=cell(0,0);
	for j = 1:nov
		variables(j,1)={B(varstart(j):varstop(j))};
	end
	
	
	
	% Replace incompatible characters in variable names (e.g., '-' can't be in MATLAB variable name, would throw error later on)
	for j = 1:nov
		cv    = variables{j}; newcv = cv; % Current variable
		F=strfind(cv,'-'); newcv(F)='_';
		F=strfind(cv,'/'); newcv(F)='D';
		F=strfind(cv,'+'); newcv(F)='A'; % Add more conversions here, if you need them
		% If any changes have been made to variable name then mention that and store the new name
		if strcmp(cv,newcv)~=true
			disp(cat(2,'       Variable "',cv,'" renamed to ',newcv,'.'))
			variables{j}=newcv;				
		end
	end



	% Read variables' units (not really useful, but anyways)
	B=fgetl(fid);
	unitstart=[1 strfind(B,',')+1];
	unitstop =[strfind(B,',')-1 size(B,2)];
	nou=length(unitstart); % Number of units
	disp(cat(2,'    Number of units found:     ',int2str(nov)));
	for j = 1:nou
		units(j,1)={B(unitstart(j):unitstop(j))};
	end

	
	
	% Build the string that defines how to read the datafile.
	disp('    Determining which variables to load...');
	nov=length(variables);
	string='%s'; % First is always cruiseID
	for j = 2:nov %
		cv=variables{j}; % current variable
		if strcmp(cv,'SECT_ID')
			string=[string '%*s'];
			variables(j)={'noload'};
			disp(cat(2,'       Variable "',cv,'"  will not be loaded.'))			
		elseif strcmp(cv,'BTLNBR')
			string=[string '%*s'];
			variables(j)={'noload'};
			disp(cat(2,'       Variable "',cv,'"  will not be loaded.'))			
		elseif strcmp(cv,'SAMPNO')
			string=[string '%*s'];
			variables(j)={'noload'};
			disp(cat(2,'       Variable "',cv,'"  will not be loaded.'))
        % for pacX
        elseif strcmp(cv,'PH_SCALE')
            string=[string '%*s'];
            variables(j)={'noload'};
            disp(cat(2,'       Variable "',cv,'"  will not be loaded.'))
        elseif strcmp(cv,'SECT')
            string=[string '%*s'];
            variables(j)={'noload'};
            disp(cat(2,'       Variable "',cv,'"  will not be loaded.'))
        %%%
		elseif any(strmatch(cv,TRASHLIST,'exact'));
			string=[string '%*f'];
			variables(j)={'noload'};
			disp(cat(2,'       Variable "',cv,'" is on the TRASH-LIST, so will not be loaded.'))			
		elseif strcmp(cv,'STNNBR')
			string=[string '%s'];
		elseif strcmp(cv,'BTLNBR_FLAG_W')
			string=[string '%f'];
		elseif any(strmatch(cv,OKLIST,'exact')); 	
			string=[string '%f'];
		elseif any(strmatch(cv,MERGELIST,'exact'))
			aaa=rem(strmatch(cv,MERGELIST,'exact'),size(MERGELIST,1)); % Determine on which row of M the hit is.
			if aaa==0; aaa=size(MERGELIST,1); end
			variables(j)=MERGELIST(aaa); % Rename the variable to the corresponding name in the first column of MERGELIST
			string=[string '%f'];
			if strcmp(cv,MERGELIST(aaa))==0
				disp(cat(2,'       Variable "',cv,'" will be loaded and renamed to "',MERGELIST{aaa},'".'));
			end
		else
			disp(cat(2,'**************      dude wtf this var is on no lists... : ',cv))
			keyboard
		end
	end
	string=[string '\n']; % Add the end-of-line specifier to read string
	variables(strmatch('noload',variables))=[]; % Scrub the "noload" lines
	

	
	
	% Check if the loaded variables are already present, else pre-allocate
	disp('    Now checking for new variables...');
	nov=length(variables);
	for j = 1:nov %
		if exist(variables{j},'var')==0
			disp(cat(2,'       Variable "',variables{j},'" does not yet exist. Preallocating now.'))
			cv=variables{j}; % current variable	
			eval(cat(2,cv,'=nan(',int2str(pal),',1);'));
			allvariables(end+1,1)=variables(j);	% Store the name of the variable
			allunits(end+1,1)=units(j);	% Store the name of the variable
		end			
	end



	% Load the data
	disp('    Now reading data...');
	[C,bytesread]=textscan(fid,string,'delimiter',','); % Read data using the read string
	% If the not-first columns are 1 element shorter than the first (which has the 'END_DATA' flag), scrub that word, in order to make C rectangular again
	if size(C{1},1)==size(C{2},1)+1
		C{1}=C{1}(1:end-1);
    end
    nos=size(C{1},1)-1;
	disp(cat(2,'       Variables loaded: ',int2str(size(C,2)),'.'));
	disp(cat(2,'       Samples   loaded: ',int2str(nos),'.'));
	if bytesread<filesize*.9
		disp('***** File not entirely read? Holding... type "return" or "dbquit". ');
		keyboard
	end



	% remove all alphanumeric data from C{2}, the stationnumbers, and change to double - THIS IS NOT VERY PRETTY YET, SORRY
	% c=unique(C{2}); % for cnaX
	newsns=C{2};
    % for pacX
    for j = 1:length(newsns)
        F=strfind(newsns{j},'_'); newsns{j}(F)='';
        F=strfind(newsns{j},'-'); newsns{j}(F)='';
        F=strfind(newsns{j},'('); newsns{j}(F)='';
        F=strfind(newsns{j},')'); newsns{j}(F)='';
        oldsn=newsns{j};
        newsn = '';
        for k = 1:length(oldsn)
            foo = oldsn(k);
            if isletter(oldsn(k))
                foo = num2str(double(oldsn(k)));
            end
            newsn = strcat(newsn,foo);
        end
        newsns{j}=newsn;
    end
    % debug
    %unique(newsns)
    %%%
    %%% for cnaX
    %c=unique(newsns);
	%for j = 1:length(c)
    %    oldsn=c{j};
    %    F=isletter(oldsn);
	%	if any(F)
	%		newsn=oldsn(~F);
	%		F=strmatch(oldsn,newsns);
	%		newsns(F)={newsn};
    %    end
    %end
	newsns=str2num(char(newsns));
	if length(C{2})>length(newsns)
		newsns=[newsns;nan];
    end	
	C{2}=newsns;


	% Assign variable names to data
	writeto=writefrom+nos-1;
	disp(cat(2,'    Now storing values (in positions ',int2str(writefrom),' to ',int2str(writeto),')...'));
	cruiseID(writefrom:writeto)=i; % Store the cruiseID's
	allexpocodes(i,1)=C{1}(1);     % Store the expocode (from first row of first column of datafile)
	for j = 1:nov
		D=C{j}; 
		D=D(1:end-1); % Don't store the last row (which is empty except for reading 'END_DATA')
		cv=variables{j}; % current variable
        % debug for pacX
        % disp(cat(2,' check: ', int2str(j), ':', cv, ':', int2str(size(D)), ':', int2str(nos)));
 		%%%
        eval(cat(2,cv,'(',int2str(writefrom),':',int2str(writeto),')=D;'))
	end
	writefrom=writefrom+nos; % Update 'writefrom', so that next cruise will be appended to this one



	% User feedback
	disp('    ***')
	disp(cat(2,'    Total number of variables in memory so far: ',int2str(length(allvariables)),'.'));
	disp(cat(2,'    Total number of datalines in memory so far: ',int2str(writeto),'.'));
	disp(cat(2,'    Pre-allocated space for variables is ',int2str((writeto./pal)*100),'% full.'))
	
	if int2str(writeto)>0.9*pal; % If the preallocated space if over 90% full, alert user.
		disp('       PREALLOCATED SPACE FOR VARIABLES IS 90 % FULL! Stopping.')
		return
	end
	if length(allvariables)>100 & noalert==0; % If more than 100 variables have been loaded, consult user
		disp('     More than 100 unique variables have been loaded. Continuing may cause memory problems.')
		disp('     Type "dbquit" to cancel, or "return" to continue.')
		keyboard
		noalert=1;
	end
	disp(' '); disp(' '); disp(' '); disp(' ');
    
    fclose(fid);
	
end
disp(cat(2,'*** Finished loading and merging. Total number of samples: ',int2str(writeto),'. Total number of variables: ',int2str(length(allvariables)),'.'));



% %%%%%%%%%%%%%%%%%%%%%%%%%   THE DATA PROCESSING BIT  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

% keyboard

% Remove trailing (overallocated) nans and replace MVI's (-999's and -9's) with NaNs
disp('*** Now removing overallocated space, setting MVIs to NaN, et cetera');
nos=writeto; % Number of samples (equals the last position written to)
EXPOCODE=EXPOCODE(1:nos);
cruiseID=cruiseID(1:nos);
TIME    =TIME(1:nos);
for i = 1:length(allvariables)
	cv=allvariables{i}; % current variable	

	eval(cat(2,cv,'=',cv,'(1:',int2str(nos),');')) % Remove trailer rows
	eval(cat(2,cv,'(',cv,'==-999)=nan;'));         % Replace -999's with NaNs

	% If considering latitude, longitude or DELC14 (where -9 might be a real value), skip to next variable.
	if strcmp(cv,'LATITUDE')==1 | strcmp(cv,'LONGITUDE')==1 | strcmp(cv,'DELC14')==1; continue; end
	
	eval(cat(2,cv,'(',cv,'==-9)=nan;'));           % Replace -9's with NaNs

end



% Generate a variable holding the date/time in a MATLAB compatible format
% First, convert to strings.
disp('*** Now combining DATE and TIME vectors into a new, MATLAB-compatible vector "DATETIME".');
a=num2str(DATE); 
b=num2str(TIME);
% Then, pad the time vector with zeroes (otherwise, ten past midnight reads "  10" in stead of "0010")
b(b==32)=48; % Replace spaces (ASCII#32) with zeroes (ASCII#48)
c=[a b];
DATETIME=datenum(c,'yyyymmddHHMM');



% Generate excel-proof columns for date and time (optional)
% The next two lines work only if you've got the 'finance' toolbox installed. 
% That's why it checks for the availability of those functions.
% If this, for whatever reason, isn't foolproof, just delete/comment these 6 lines
if exist('year') & exist('month') & exist('day')
	disp('*** Generating individual columns for year, month et cetera.');	
	YEAR = year(DATETIME); MONTH = month(DATETIME);	DAY = day(DATETIME); HOUR = hour(DATETIME);	MINUTE = minute(DATETIME);
	allvariables=[allvariables;{'YEAR';'MONTH';'DAY';'HOUR';'MINUTE'}]; % Add five new variables
	allunits    =[allunits;    {'';'';'';'';''}]; % And add their units	
end



% Copy all individual variables into one big array
disp('*** Copying all individual variables into one big array "DATA".')
noav = length(allvariables);
DATA=nan(nos,noav);
for i = 1:noav 
	disp([i noav])
	cv=allvariables{i}; % current variable	
	eval(cat(2,'DATA(:,i)=',cv,';'))
end



% Count how many samples of each parameter are available for each cruise
disp('*** Determining total amounts of samples for each variable, for each cruise.')
sums=nan(nof,noav);
for i = 1:nof
	F=cruiseID==i;
	for j = 1:noav 
		cv=allvariables{j}; % current variable
		eval(cat(2,'sums(i,j)=sum(~isnan(',cv,'(F)));'));
	end
end






% %%%%%%%%%%%%%%%%%%%%%%%%%   THE DATA SAVING BIT  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


% Write the big array to an excel compatible csv file (somewhat resembling exchange format, actually)
if savespreadsheet==1
	disp('*** Generating .csv file of merged data, for use in Excel.')
	fid=fopen('output/tempcsv.csv','w');
	% First, write some text above the tables
	fprintf(fid,cat(2,'### Merge performed on ',datestr(now),' using excread version ',excversion,'.\n'));
	% Then, write the table's header
	fprintf(fid,'EXPOCODE');
	for i = 1:noav
		fprintf(fid,',%s',char(allvariables{i}));
	end
	fprintf(fid,'\n');
	for i = 1:noav
		fprintf(fid,',%s',char(allunits{i}));
	end
	fprintf(fid,'\n');
	% Then, write the table's contents
	% Generate write-string
	string='';
	for j = 1:noav
		string=[string ',%f'];
	end
	string=[string '\n']; % Add the end-of-line specifier to write string
	for i = 1:nos;
		if ismember(i,round(1:nos/10:nos))
			disp(cat(2,'      ',int2str((i/nos)*100),'% done'))
		end
		j=cruiseID(i);
		fprintf(fid,'%s',char(allexpocodes(j)));
		fprintf(fid,string,[DATA(i,:)]);
	end
	disp('     100% done')
else
	disp('*** Skipping .csv output.')
end

% Write lookuptable & valuecounts
format long
disp('*** Generating lookup table.');
fid=fopen('output/templookup.csv','w');
% First, write some text above the tables
fprintf(fid,'### This file holds the EXPOCODE-cruiseID relation and the variable counts of the\n');
fprintf(fid,cat(2,'### merge performed on ',datestr(now),' using EXCREAD version ',excversion,'.\n'));
% Then, write the table's header ('expocode'+everything in 'allvariables')
fprintf(fid,'EXPOCODE      ');
for i = 1:noav
	fprintf(fid,',%14s',char(allvariables{i}));
end
fprintf(fid,'\n'); % and add the EOL character
% Then, write the table's contents
string=',%14d'; 
for j = 1:noav-1 % 1 shorter, since the first one is defined in the previous line
	string=[string ',%14d'];
end
string=[string '\n']; % Add the end-of-line specifier to read string
% Now write, column per column
for i = 1:nof
	fprintf(fid,'%14s',char(allexpocodes{i})); % Write the expocode in the first column
	fprintf(fid,string,[i sums(i,2:end)]); % Don't write the first column of sums ('cruiseID'), but rather write the cruiseID 'i' itself
end



% Check for existing .mat files (previous output?) and determine name of new output
outputnumber=1;
while exist(cat(2,'output/',int2str(outputnumber),'-BIGARRAY.mat'),'file')~=0
	outputnumber=outputnumber+1;
end 
savename1  = cat(2,'output/',int2str(outputnumber),'-BIGARRAY.mat');
savename2  = cat(2,'output/',int2str(outputnumber),'-VECTORS.mat');
csvname    = cat(2,'output/',int2str(outputnumber),'-spreadsheet.csv');
logname    = cat(2,'output/',int2str(outputnumber),'-mergelog.txt');
lookupname = cat(2,'output/',int2str(outputnumber),'-lookuptable.csv');



% Clean up and save
DATAVARIABLES=allvariables
DATAUNITS    =allunits;
DATAEXPOCODES=allexpocodes
clear A B C D E F G writefrom writeto nov nou string unitstart unitstop varstart excversion allunits allvariables noav sums
clear varstop a b c bytes fid file i j lpmb nof nos pal stillheader units variable variables cv sidp noav outputnumber EXPOCODE ans 
clear newcv newsn newsns noalert oldsn Tbytes bytesread filesize TRASHLIST aaa OKLIST MERGELIST

% File 1
save(savename1,'DATA*'); clear DATA* % Save and clear big array and associated vectors
disp(cat(2,'*** Final matlabfile (1 big array) saved as: "'       ,savename1,'".'));

% File 2
save(savename2);                     % Save remainder of workspace (that is, the loose vectors)
disp(cat(2,'*** Final matlabfile (individual vectors) saved as: "',savename2,'".'));

% File 3
if savespreadsheet==1; 
	copyfile('output/tempcsv.csv'   ,csvname);  % Copy csvfile to final name 
	disp(cat(2,'*** Final csv (excel) file saved as: "',csvname,'".')); 
	delete('output/tempcsv.csv');                 % Delete temporary csvfile
end

% File 4
copyfile('output/templookup.csv',lookupname); % Copy lookupfile to final name
disp(cat(2,'*** Associated lookuptable saved as: "',lookupname ,'".'));
delete('output/templookup.csv');              % Delete temporary lookupfile

% File 5 (timefraud here)
disp(cat(2,'*** Associated     logfile saved as: "',logname ,'".'));

% Done!
disp(cat(2,'*** DONE! --- That took ',int2str(toc),' seconds.')); disp(' ')

diary off
copyfile('output/templog.txt'   ,logname);    % Copy logfile    to final name
delete('output/templog.txt');                 % Delete temporary logfile

clear lookupname newcv newsn newsns noalert oldsn Tbytes savename1 savename2 bytesread savespreadsheet cruiseID csvname filesize logname 