function varargout = ma_make_database(varargin)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% this function reads in the file brain_areas.txt which must be a tab
%% delimited text file with the columns: [area,ba,func,tx,ty,tz,pmid]
%% it connects to pubmed to find the details of each paper and saves them
%% in a file called pubmed_data in a structure called pubmed
%% this function only needs to be run when the raw data has been updated.

global ma_defaults 

if(length(ma_defaults)==0)
    load ma_defaults
    disp('loading defaults')
end

if(nargin>0)
    action = varargin{1};
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

switch lower(action)
    case 'read_brain_areas'
        
        [garea,gbrod,gfunc,gxx,gyy,gzz,gcode,gdescrip,gpmid] = ...
            textread([ma_defaults.swd,filesep,ma_defaults.globaldb], ...
            '%s%s%s%n%n%n%s%s%d','delimiter',',','emptyvalue',NaN);
        try
            [larea,lbrod,lfunc,lxx,lyy,lzz,lcode,ldescrip,lpmid] = ...
                textread([ma_defaults.swd,filesep,ma_defaults.localdb], ...
                '%s%s%s%n%n%n%s%s%d','delimiter',',');        
        catch
            disp('No local database found')
            larea = []; lbrod = [];  lfunc = [];
            lxx = [];  lyy = []; lzz = [];
            lcode = {}; ldescrip = []; lpmid = [];
        end
                   
        %% run some checks
        glen = [length(garea),length(gbrod),length(gxx),length(gcode),length(gdescrip),length(gpmid)];
        if(std(glen)>0)
            disp('There is a problem with the global database')
            fields = {'area ','brod ','xyz ','code ','descrip ','pmid '}
            disp(['There are ',num2str(max(glen)),' entries in these columns: ', ...
                    cat(2,fields{find(glen==max(glen))})]);
            disp(['and ',num2str(min(glen)),' entries in these columns: ',...
                    cat(2,fields{find(glen==min(glen))})]);
        end

        ba.area = [garea;larea];
        ba.brod = [gbrod;lbrod];
        ba.func = [gfunc;lfunc];
        xyz = [ [gxx,gyy,gzz];[lxx,lyy,lzz] ];
        csys = cat(1,[gcode{:} ; lcode{:}]);
        cc = csys=='T';
        ba.txyz = [];
        ba.txyz(cc,:) = [xyz(cc,:)];
        ba.txyz(cc==0,:) = mni2tal(xyz(cc==0,:));
        ba.mxyz = tal2mni(ba.txyz);
        
        ba.descrip = [gdescrip;ldescrip];
        ba.pmid = [gpmid;lpmid];
        ba.nglobal = length(gxx);
        ba.nlocal = length(lxx);
        
        
        varargout{1} = ba;
        
    case 'rebuild'
        
        if(ma_interactive('yesno','Really rebuild the database?')==0)
            return;
        end
        
        disp('Rebuilding the whole database from scratch, a bit slow')
        
        ba = ma_make_database('read_brain_areas');
        
        ids = unique(ba.pmid);
        ids = ids(ids>0);
        clear pubmed
        
        for i=1:length(ids)
            pubmed(i) = read_pubmed(ids(i));
            ma_interactive('progress',round(i/length(ids)),'Building database');
        end
        
        save pubmed_data pubmed
        ma_make_database('stats')
        disp('new pubmed_data file saved successfully')
        
    case 'check'
        ba = ma_make_database('read_brain_areas');

        ids = unique(ba.pmid);
        ids = ids(ids>0);
        load pubmed_data
        
        oldids = cat(1,pubmed.id);
        newids = setdiff(ids,oldids);
        varargout = length(newids);
        
    case 'update'
        
        ba = ma_make_database('read_brain_areas');

        ids = unique(ba.pmid);
        ids = ids(ids>0);
        load pubmed_data
        
        oldids = cat(1,pubmed.id);
        newids = setdiff(ids,oldids);
        disp(['Updating ',num2str(length(newids)),' papers in database'])
        ma_interactive('disp_text',{['Updating ',num2str(length(newids)),' papers in database']})

        for i=1:length(newids)
            pubmed(end+1) = read_pubmed(newids(i));
        end
        
        length(pubmed)
        
        save pubmed_data pubmed
        ma_make_database('stats')
        disp('new pubmed_data file saved successfully')
        
    case 'stats'
        
        ba = ma_make_database('read_brain_areas');

        ids = unique(ba.pmid);
        dirba = dir('brain_areas.txt');
        dirpm = dir('pubmed_data.mat');
        info = {['----- DATABASE STATISTICS ---------'],[' '], ...
            ['Coordinate table is: brain_areas.txt'], ...
            ['which has ',num2str(length(ba.area)),' unique entries'], ...
            ['and was last saved on ',dirba.date], ...
            [' '], ...
            ['Pubmed database is: pubmed_data.mat'], ...
            ['which has ',num2str(length(ids)),' papers'], ...
            ['and was last saved on ',dirpm.date]};
        
        for i=1:length(info)
            disp(info{i});
        end

        ma_interactive('disp_text',info)
                
end
                
%---------------------------------------------------------
% get information from pubmed
function pdata = read_pubmed(pmid)

pdata = {};

%% load up defaults
global ma_defaults;
tofind = ma_defaults.db.tofind;
unicodes = ma_defaults.db.unicodes;

base = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?';
db = 'db=pubmed';
id = 'id=';
mode = 'retmode=xml';
email = 'email=antonia.hamilton@gmail.com';
tool = 'tool=fmri_meta_anlysis_database';

s = urlread([base,db,'&',id,num2str(pmid),'&',mode,'&',email,'&',tool]);
    
uu = strfind(s,'&#');
%% look for unicodes
while length(uu)>0
    %% try replacing
    for j=1:size(unicodes,1)
        s = regexprep(s,unicodes{j,1},unicodes{j,2});
    end
    
    uu = strfind(s,'&#');
    if(length(uu)>0)  %% if unidentified unicodes
        uu = uu(1);
        disp(s((-30:30)+uu))
        correct = input('Enter what this should be ','s');
        code = s(uu:uu+7);
        unicodes{end+1,1} = code;
        unicodes{end,2} = correct;
        
        ma_defaults.db.unicodes = unicodes;
        save ma_defaults ma_defaults;
        disp('Unicodes updated')
    end
end
    
%% interpret everything
pdata.id = pmid;
for j=1:length(tofind)
    first = strfind(s,tofind{j,2})+length(tofind{j,2});
    last = strfind(s,tofind{j,3})-1;
    
    if(strmatch(tofind{j,1},'authors'))
        for k=1:length(first)
            eval(['pdata.authors{',num2str(k),'} = s(first(',num2str(k),'):last(',num2str(k),'));'])
        end
    else
        pdata = setfield(pdata,tofind{j,1},s(first(1):last(1)));
    end
end

%---------------------------------------------------------

    