% IMPORTIHPD : downloads IHPD data from the web and puts it into a structure.
%
% Usage:     IHPD = ImportIHPD(fname,[true/false])
%            IHPD = ImportIHPD(fname)
%
% Input: The first argument specifies the name of the excel file from the IHPD website 
% to be downloaded. The format .xlsx does not have to be included as part of the file name. 
% All file names have the same format: hp followed by the release date with the first two 
% digits indicating the year followed by 0 and an fourth digit to identify the quarter. 
% Please note that fname should be written in string format, such as 'hp1404'. 
% The second argument is optional. It specifies if a fresh copy should be downloaded from
% the internet or not. The default is true. If set to false, the program will attempt
% to locate 'fname' excel file on the local drive and determine if it is available. 
%
% Output: IHPD is a struct containing years, country names, and all the
% variables from the international house price database (IHPD) as fields.
%
% Data provider:
%   We would appreciate that anyone wishing to use this dataset, modified or otherwise, 
%   acknowledge the source of the data publicly available through this website with a 
%   citation of the working paper: for example, including a statement such as: “The authors 
%   acknowledge use of the dataset described in Mack and Martínez-García (2011).”
%   http://www.dallasfed.org/institute/houseprice/
%
% Contacting Author: Ayse Kabukcuoglu (Koç University)
% E-mail: akabukcuoglu@ku.edu.tr
% Author: Enrique Martinez-Garcia (Federal Reserve Bank of Dallas)
% E-mail: enrique.martinez-garcia@dal.frb.org
% The code is a straightforward adaptation of the ImportPWT.m code produced to
% import Penn World Table by:
%   Author: Yvan Lengwiler, University of Basel, Switzerland
%   url   : http://www.wwz.unibas.ch/lengwiler
%   email : yvan.lengwiler@unibas.ch (please write if you need help)
% Date  : 9/22/2015

%%%%%%%%%%%%%%%%%%%%%%%%% PRACTICAL EXAMPLES %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% % download the data from IHPD for the 2014:Q4 release of the database
% %IHPD = ImportIHPD('hp1404',false); % if the file has already been saved in the local drive 
% IHPD = ImportIHPD('hp1404'); % if the file must be downloaded from the website
% % collect data
% us = find(ismember(IHPD.countries,'US'));  % find index of US
% price_to_income_ratio = IHPD.HPI(:,us)./IHPD.PDI(:,us);
% % create dates
% years = unique(IHPD.dates(:,1)); 
% numyears = length(years); 
% jan1s = zeros(numyears,1); 
% quarters = zeros(4,numyears);
% numyears = length(years); 
% for Yearno = 1:numyears 
% thisyear = years(Yearno); 
% jan1s(Yearno) = datenum([thisyear 1 1]); 
% quarters(1,Yearno) = jan1s(Yearno); 
% quarters(2,Yearno) = datenum([thisyear 4 1]); 
% quarters(3,Yearno) = datenum([thisyear 7 1]); 
% quarters(4,Yearno) = datenum([thisyear 10 1]); 
% end
% quarters=quarters(:);
% % plot the growth rate for the price to income ratio series
% figure('Name','The United States''s growth rate of the price to income ratio');
% plot(quarters(2:end,1),100*diff(log(price_to_income_ratio))); 
% set(gca,'XTick', jan1s); 
% datetick('x', 'yyyy');
% % start new figure
% figure('Name','USA vs Spain');
% spain = find(ismember(IHPD.countries,'Spain'));  % find index of Spain
% % plot USA's vs Spain's house price to income ratio
% comparison_price_to_income = IHPD.HPI(:,[us,spain])./IHPD.PDI(:,[us,spain]);
% plot(quarters(:),comparison_price_to_income,'LineSmoothing','on','LineWidth',1.5);
% set(gca,'XTick', jan1s); 
% datetick('x', 'yyyy');
% legend('USA','Spain');
% legend('location','SouthEast');
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

function IHPD = ImportIHPD(xlsxfilename,varargin)

fname = strcat(xlsxfilename,'.xlsx');
    % should file be downloaded?
    if nargin == 1
        doDownload = true;
    else
        doDownload = varargin{1};
    end
    
    % download or locate file
    if doDownload
        
       % download data from IHPD website
       url = strcat('http://www.dallasfed.org/assets/documents/institute/houseprice/',fname);
       [fname, success] = urlwrite(url,fname);
        if ~success
            error('IHPD:download_failed', ...
                ['Tried to download from this URL, ''%s''.\n', ...
                'Download failed for some reason.\n'], url);
        else
            disp('File successfully downloaded from IHPD site.');
        end
        
    else
        
        % use local copy of IHPD file
        fprintf('Looking for local version of ''%s''\n', fname);
        d = dir(fname);
        if isempty(d)
            fprintf('failed.\n')
            error('IHPD:search_failed', ...
                'Could not find local copy of ''%s''\n', fname);
        else
            fprintf('found it.\n');
        end
        
    end
    
     % read Excel file
     [HPIdata,HPItxt] = xlsread(fname,'HPI');
     [RHPIdata,RHPItxt] = xlsread(fname,'RHPI');
     [PDIdata,PDItxt] = xlsread(fname,'PDI');
     [RPDIdata, RPDItxt] = xlsread(fname,'RPDI');
     vars = {'HPI','RHPI','PDI','RPDI'}; data=cell(1,numel(vars));
     data{1} = HPIdata(:,[1:end-2,end]); data{2} = RHPIdata(:,[1:end-2,end]); % we skip one column between all the countries and the aggregate at the end because it is empty
     data{3} = PDIdata(:,[1:end-2,end]); data{4} = RPDIdata(:,[1:end-2,end]);
     country = HPItxt(1,[2:end-2,end]); 
     N = length(country);
     years = HPItxt(3:end, 1);  
     formatIn = 'YYYY:QQ';
     dates = datenum(years,formatIn); 
     dates = datevec(dates);
     year = dates(:,1); col2=dates(:,2); 
     quarter((col2==1)) = 1;
     quarter((col2==4)) = 2;
     quarter((col2==7)) = 3;
     quarter((col2==10)) = 4;
     dates = [year quarter'];
     T = length(year);
    % create output variable
     IHPD = struct( ...
        'T',        T, ...
        'dates',     dates, ...
        'N',        N, ...
        'countries',  {country});
   
     fprintf(' %i countries and their aggregate, from %i Q%i to %i Q%i.\n',N-1,dates(1,1),dates(1,2),dates(end,1),dates(end,2));
    
     % now add all the data
     
     nvars = numel(vars);
     for k = 1:numel(vars)
         dataset = data{k};    % rows are dates, columns are countries
         name = genvarname(vars{k});         % make sure variable name is legal
         fprintf(' Measure #%i: %s (%ix%i)\n',k,name,size(dataset))
         IHPD.(name) = dataset;                  % add variable to the struct
     end



    disp('done.');
    


end