function varargout = ma_make_database(varargin)
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%% this function reads in the file brain_areas.txt which must be a tab
%% delimited text file with the columns: [area,ba,func,tx,ty,tz,pmid]
%% it connects to pubmed to find the details of each paper and saves them
%% in a file called pubmed_data in a structure called pubmed
%% this function only needs to be run when the raw data has been updated.

global ma

if(nargin>0)
    action = varargin{1};
end

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

switch lower(action)
    case 'read_brain_areas'
        
        try
            [garea,gbrod,gfunc,gxx,gyy,gzz,gcode,gdescrip,gpmid] = ...
                textread([ma.defaults.globaldb], ...
                '%s%s%s%n%n%n%s%s%d','delimiter',',','emptyvalue',NaN);
        catch
            disp('Global database could not be read, error:')
            disp(lasterr)
            disp(['Check that ',ma.defaults.globaldb,' is on the current path'])
            varargout = -1;
            return;
        end
        
        try
            [larea,lbrod,lfunc,lxx,lyy,lzz,lcode,ldescrip,lpmid] = ...
                textread([ma.defaults.localdb], ...
                '%s%s%s%n%n%n%s%s%d','delimiter',',');        
        catch
            disp('No local database found, continuing with global only')
            larea = []; lbrod = [];  lfunc = [];
            lxx = [];  lyy = []; lzz = [];
            lcode = {}; ldescrip = []; lpmid = [];
        end
                   
        %% run some checks
        glen = [length(garea),length(gbrod),length(gxx),length(gcode),length(gdescrip),length(gpmid)];
        if(std(glen)>0)
            disp('There is a problem with the global database')
            fields = {'area ','brod ','xyz ','code ','descrip ','pmid '}
            disp(['There are ',num2str(max(glen)),' entries in these columns: ', ...
                    cat(2,fields{find(glen==max(glen))})]);
            disp(['and ',num2str(min(glen)),' entries in these columns: ',...
                    cat(2,fields{find(glen==min(glen))})]);
            varargout{1} = -1;
            return
        end
        if(length(larea)>0)
            llen = [length(larea),length(lbrod),length(lxx),length(lcode),length(ldescrip),length(lpmid)];
            if(std(llen)>0)
                disp('There is a problem with the local database')
                fields = {'area ','brod ','xyz ','code ','descrip ','pmid '}
                disp(['There are ',num2str(max(llen)),' entries in these columns: ', ...
                        cat(2,fields{find(llen==max(llen))})]);
                disp(['and ',num2str(min(llen)),' entries in these columns: ',...
                        cat(2,fields{find(llen==min(llen))})]);
                varargout{1} = -1;
                return
            end
        end

        %% put information into ma structure
        ma.ba.area = [garea;larea];
        ma.ba.brod = [gbrod;lbrod];
        ma.ba.func = [gfunc;lfunc];
        xyz = [ [gxx,gyy,gzz];[lxx,lyy,lzz] ];
        csys = [cat(1,gcode{:}) ; cat(1,lcode{:})];
        cc = csys=='T';
        ma.ba.txyz = [];
        ma.ba.txyz(cc,:) = [xyz(cc,:)];
        ma.ba.txyz(cc==0,:) = mni2tal(xyz(cc==0,:));
        ma.ba.mxyz = tal2mni(ma.ba.txyz);
        
        ma.ba.descrip = [gdescrip;ldescrip];
        ma.ba.pmid = [gpmid;lpmid];
        ma.ba.nglobal = length(gxx);
        ma.ba.nlocal = length(lxx);
        
        varargout{1} = 1;
                
    case 'rebuild'
        
        if(ma_interactive('yesno','Really rebuild the database?')==0)
            'no'
            return
        end
        'yes'
        disp('Rebuilding the whole database from scratch, a bit slow')
        
        ids = unique(ma.ba.pmid);
        ids = ids(ids>0);
        clear pubmed
        pubmed = read_pubmed(ids(1));
        for i=2:length(ids)
            pubmed(end+1) = read_pubmed(ids(i));
            ma_interactive('progress',i/length(ids),'Building database');
        end
        
        save(ma.defaults.pubmeddb,'pubmed','-v6')
        ma.pubmed = pubmed;
        ma_database('stats')
        disp('new pubmed_data file saved successfully')
        
    case 'check'
        ids = unique(ma.ba.pmid);
        ids = ids(ids>0);
        
        oldids = cat(1,ma.pubmed.id);
        newids = setdiff(ids,oldids);
        varargout{1} = length(newids);
        
    case 'update'
        
        ids = unique(ma.ba.pmid);
        ids = ids(ids>0);
        
        oldids = cat(1,ma.pubmed.id);
        newids = setdiff(ids,oldids);
        disp(['Updating ',num2str(length(newids)),' papers in database'])
        ma_interactive('disp_text',{['Updating ',num2str(length(newids)),' papers in database']})

        pubmed = ma.pubmed;
        for i=1:length(newids)
            pubmed(end+1) = read_pubmed(newids(i));
        end
        
        save(ma.defaults.pubmeddb,'pubmed','-v6')
        ma.pubmed = pubmed;
        ma_database('stats')
        disp('new pubmed_data file saved successfully')
        
    case 'stats'
        
        %% check to see if the ba database and pubmed database agree
        diff = ma_database('check');

        ids = unique(ma.ba.pmid);
        dirglobal = dir(ma.defaults.globaldb);
        dirlocal = dir(ma.defaults.localdb);
        dirpm = dir(ma.defaults.pubmeddb);
        info = {['----- DATABASE STATISTICS ---------'],[' '], ...
                ['Global Coordinate table is: ',ma.defaults.globaldb], ...
                ['which has ',num2str(ma.ba.nglobal),' entries'], ...
                ['and was last saved on ',dirglobal.date], ...
                [' '], ...
                ['Local Coordinate table is: ',ma.defaults.localdb], ...
                ['which has ',num2str(ma.ba.nlocal),' entries'], ...
                ['and was last saved on ',dirlocal.date], ...
                [' '], ...
                ['Pubmed database is: ',ma.defaults.pubmeddb], ...
                ['which has ',num2str(length(ids)),' papers'], ...
                ['and was last saved on ',dirpm.date]};
        
        if(diff>0)
            info = [info,{[' '],...
                        [num2str(diff),' papers in coordinate database are not in pubmed database'], ...
                        ['You should update the pubmed database now']}]
        else
            info = [info,{[' '],...
                        ['Database checks passed']}]
        end
        
        for i=1:length(info)
            disp(info{i});
        end

        ma_interactive('disp_text',info)
                
end
                
%---------------------------------------------------------
% get information from pubmed
function pdata = read_pubmed(pmid)

pdata = {};

%% load up defaults
global ma;
tofind = ma.defaults.tofind;
unicodes = ma.defaults.unicodes;

base = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?';
db = 'db=pubmed';
id = 'id=';
mode = 'retmode=text';
email = 'email=antonia.hamilton@gmail.com';
tool = 'tool=fmri_meta_anlysis_database';

str = [base,db,'&',id,num2str(pmid),'&',mode,'&',email,'&',tool];
s = urlread(str);
    
try
    ob = strfind(s,'{');
    cb = strfind(s,'}');
    %% find authors
    aa = strfind(s,'authors');
    oa = ob(ob>aa(1));
    ca = cb(cb>aa(1));
    aut = s(oa(1):ca(1));
    quo = strfind(aut,'"');
    for i=1:2:length(quo)
        pdata.authors{(i+1)/2} = aut(quo(i)+1:quo(i+1)-1);
    end

    %% find journal
    quo = strfind(s,'"');
    jquo = quo(quo>strfind(s,'ml-jta'));
    pdata.journal = s(jquo(1)+1:jquo(2)-1);

    %% find year
    yy = strfind(s,'year');
    pdata.year = str2num(s(yy(1)+(4:9)));

    %% find title
    cit = strfind(s,'cit');
    oa = ob(ob>cit(1));
    ca = cb(cb>cit(1));
    title = s(oa(1):ca(1));
    quo = strfind(title,'"');
    tmp = title(quo(1)+1:quo(2)-1);
    %% get rid of spaces in title
    utmp = double(tmp);
    utmp(utmp==10) = 32;  %% replace linebreak with space
    pdata.title = char(utmp);
    
    %% keep id
    pdata.id = pmid;
catch
    spm('beep')
    disp('Problem reading data from pubmed')
    disp('Please report this bug to Antonia')
    keyboard
end
if 0  %% old version for using xml mode
    uu = strfind(s,'&#');
    %% look for unicodes
    while length(uu)>0
        %% try replacing
        for j=1:size(unicodes,1)
            s = regexprep(s,unicodes{j,1},unicodes{j,2});
        end

        uu = strfind(s,'&#');
        if(length(uu)>0)  %% if unidentified unicodes
            uu = uu(1);
            disp(s((-30:30)+uu))
            correct = input('Enter what this should be ','s');
            code = s(uu:uu+7);
            unicodes{end+1,1} = code;
            unicodes{end,2} = correct;

            ma_defaults.db.unicodes = unicodes;
            save ma_defaults ma_defaults -v6;
            disp('Unicodes updated')
        end
    end

    %% interpret everything
    pdata.id = pmid;
    for j=1:length(tofind)
        first = strfind(s,tofind{j,2})+length(tofind{j,2});
        last = strfind(s,tofind{j,3})-1;

        if(strmatch(tofind{j,1},'authors'))
            for k=1:length(first)
            eval(['pdata.authors{',num2str(k),'} = s(first(',num2str(k),'):last(',num2str(k),'));'])
        end
    else
        pdata = setfield(pdata,tofind{j,1},s(first(1):last(1)));
        end
    end

    %---------------------------------------------------------
end
    