heights1976 heights1976 - 3 months ago 11
R Question

Efficiently and selectively combining columns in R

I have the following data

countrycols = alljson[,c("country_gc_str","country_ipapi_str","country_tm_str")]

head(countrycols)
country_gc_str country_ipapi_str country_tm_str
1 <NA> RU RU
2 <NA> CN CN
3 US US US
4 <NA> CD CG
5 <NA> DE DE
6 <NA> <NA> NG


I want to create a new column country_final_str which gets filled with country data in order of following preference:

country_gc_str
country_ipapi_str
country_tm_str


I am also characterizing the country income level using the following:

wbURL <- "http://api.worldbank.org/countries?per_page=304"
xmlAPI <- xmlParse(wbURL)
xmlDF <- xmlToDataFrame(xmlAPI)
xmlDF$iso2CodeChar <- as.character(xmlDF$iso2Code)
xmlDF$incomeLevelChar <- as.character(xmlDF$incomeLevel)
incomexml <- xmlDF[,c("iso2CodeChar","incomeLevelChar")]
incomexmltable <- as.data.table(incomexml)


I have the following for-loop, but it is taking forever as I have over a million records:

alljson$country_final_str <- alljson$country_gc_str
alljson$income_level <- NA

for (i in 1:length (alljson$country_final_str))
{
if (is.na(alljson$country_final_str [i]))
{
alljson$country_final_str [i] = alljson$country_ipapi_str [i];
}
if (is.na(alljson$country_final_str [i]))
{
alljson$country_final_str [i] = alljson$country_tm_str [i];
}

a<-incomexmltable[iso2CodeChar==alljson$country_final_str [i]]$incomeLevelChar

if(length(a)==0)
{
alljson$income_level [i] <- NA
} else {
alljson$income_level [i] <- a
}
}


Any thoughts for improving the efficiency/getting rid of the for loop? I cannot think of a way to
apply/lapply/tapply
, and I am on Windows so my efforts to parallelize my code using
doParallel
and
doSNOW
have failed.

See below by @thelatemail for correct answer to the column question. For the country income level, I performed:

allcountries <- unique(alljson$country_final_str)

alljson$country_income_str <- NA
sum(!is.na(countrycode(allcountries, "iso2c", "country.name")))
for (i in 1:length(allcountries))
{
a<-incomexmltable[iso2CodeChar==allcountries[i]]$incomeLevelChar
if(length(a)==0)
{
alljson$country_income_str[which(alljson$country_final_str==allcountries[i])] <- NA
} else {
alljson$country_income_str[which(alljson$country_final_str==allcountries[i])] <- a
}

alljson$country_income_str
}

Answer

Here's an attempt using matrix indexing after picking the first non-missing value in the three variables:

countrycols[
  cbind(
    seq_len(nrow(countrycols)), 
    max.col(replace( -col(countrycols), is.na(countrycols), -Inf))
  )
]
#[1] "RU" "CN" "US" "CD" "DE" "NG"

To explain the logic, break down each line:

-col(countrycols)
#     [,1] [,2] [,3]
#[1,]   -1   -2   -3
#[2,]   -1   -2   -3
#[3,]   -1   -2   -3
#[4,]   -1   -2   -3
#[5,]   -1   -2   -3
#[6,]   -1   -2   -3

replace( -col(countrycols), is.na(countrycols), -Inf)
#     [,1] [,2] [,3]
#[1,] -Inf   -2   -3
#[2,] -Inf   -2   -3
#[3,]   -1   -2   -3
#[4,] -Inf   -2   -3
#[5,] -Inf   -2   -3
#[6,] -Inf -Inf   -3

(colindex <- max.col(replace( -col(countrycols), is.na(countrycols), -Inf)) )
#[1] 2 2 1 2 2 3

cbind(rowindex=seq_len(nrow(countrycols)), colindex)
#     rowindex colindex
#[1,]        1        2
#[2,]        2        2
#[3,]        3        1
#[4,]        4        2
#[5,]        5        2
#[6,]        6        3

This final matrix is used to subset each row/col combination out of the original list.

Where countrycols was:

structure(list(country_gc_str = c(NA, NA, "US", NA, NA, NA), 
    country_ipapi_str = c("RU", "CN", "US", "CD", "DE", NA), 
    country_tm_str = c("RU", "CN", "US", "CG", "DE", "NG")), .Names = c("country_gc_str", 
"country_ipapi_str", "country_tm_str"), row.names = c("1", "2", 
"3", "4", "5", "6"), class = "data.frame")