Mark Boulder Mark Boulder - 2 months ago 19
Ruby Question

Unable to fetch deeply nested hash value

I have this rake task which uses rest-client to fetch some messy JSON from this API, and then uses hashie to make the code prettier.

Unfortunately I'm unable to fetch one of the deeply nested values,

productGroup
. If working correctly, it should output
:category => "Jeans"
or similar. Please see the JSON at the bottom.

This did not work:

mash.deep_fetch(:fields, 0).deep_locate(-> (key, value, object) { value.include?("product_group") }) { "ERROR: category" }


Example output:

% rake get_products
{:category=>nil, :name=>"Luxurous Jumpsuit", :image=>"http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/129579-0012.jpg", :price=>"599", :description=>"Lorem ipsum dolor"}


Example mash:

#<Hashie::Mash brand="Dr Denim" categories=[#<Hashie::Mash name="Kvinne > KLÆR > Jeans > Slim">] description="Lorem ipsum dolor." fields=[#<Hashie::Mash name="sale" value="false">, #<Hashie::Mash name="product_id_original" value="226693-7698">, #<Hashie::Mash name="gender" value="Kvinne">, #<Hashie::Mash name="artNumber" value="226693-7698">, #<Hashie::Mash name="productGroup" value="Jeans">, #<Hashie::Mash name="productStyle" value="Slim">, #<Hashie::Mash name="extraImageProductSmall" value="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/226693-7698.jpg">, #<Hashie::Mash name="productClass" value="Klær">, #<Hashie::Mash name="extraImageProductLarge" value="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/226693-7698.jpg">, #<Hashie::Mash name="sizes" value="W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W32/L32,W26/L30,W27/L30,W28/L30,W29/L30,W24/L30,W25/L30,W32/L30,W31/L30,W30/L30">, #<Hashie::Mash name="color" value="Mid Blue">] identifiers=#<Hashie::Mash sku="226693-7698"> language="no" name="Regina Jeans" offers=[#<Hashie::Mash feed_id=10086 id="2820760a-c5b2-494a-b5dd-ab713f796cb9" in_stock=1 modified=1474947357838 price_history=[#<Hashie::Mash date=1474949513421 price=#<Hashie::Mash currency="NOK" value="599">>] product_url="http://pdt.tradedoubler.com/click?a1234" program_logo="http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png" program_name="Nelly NO" source_product_id="226693-7698">] product_image=#<Hashie::Mash url="http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/226693-7698.jpg">>


get_products.rake:

# encoding: utf-8

# Gets messy JSON from other store via REST client and cleans it up with Hashie

require "rest_client"
require "hashie"

Product = Struct.new(:category, :name, :image, :price, :description)

module ProductsFromOtherStore
CATEGORIES = [
"festkjoler",
"jakker",
"jeans",
"jumpsuit",
"vesker"
]

def self.fetch
CATEGORIES.map do |category|
Tradedoubler.fetch category
end
end

# Prettify, ie. `fooBar` => `foo_bar`

def self.prettify(x)
case x
when Hash
x.map { |key, value| [key.underscore, prettify(value)] }.to_h
when Array
x.map { |value| prettify(value) }
else
x
end
end
end

class ProductsFromOtherStore::Tradedoubler
KEY = "FE34B1309AB749F1578AEE87D9D74535513F6B54"

# Products to fetch from API

LIMIT = 2

def self.fetch category
new(category).filtered_products.take(LIMIT)
rescue RestClient::RequestTimeout => e
Array.new
end

def initialize category
@category = category

# API doesn't support gender or category searches, so do some filtering based on available JSON fields

@filters = Array.new

define_filter { |mash|
mash.fields.any? { |field|
field.name == "gender" && field.value.downcase == "kvinne"
}
}

define_filter { |mash|
mash.categories.any? { |category|
category.name.underscore.include? @category
}
}
end

def define_filter(&filter)
@filters << filter
end

def filtered_products
filtered_mashes.map { |mash|
# puts mash

Product.new(
# mash.deep_fetch(:fields, 0).find { |field| field[:name] == "product_group" }[:value],
mash.deep_fetch(:fields, 0).deep_locate(-> (key, value, object) { value.include?("product_group") }) { "ERROR: category" },
mash.deep_fetch(:name) { "ERROR: name" },
mash.deep_fetch(:product_image, :url) { "ERROR: image URL" },
mash.deep_fetch(:offers, 0, :price_history, 0, :price, :value) { "ERROR: price" },
mash.deep_fetch(:description) { "ERROR: description" }
)
}
end

private
def request
response = RestClient::Request.execute(
:method => :get,
:url => "http://api.tradedoubler.com/1.0/products.json;q=#{ URI.encode(@category) };limit=#{ LIMIT }?token=#{ KEY }",
:timeout => 0.4
)
end

def hashes
ProductsFromOtherStore.prettify(JSON.parse(request)["products"])
end

def mashes
hashes.map { |hash| Hashie::Mash.new(hash) }.each do |mash|
mash.extend Hashie::Extensions::DeepFetch
mash.extend Hashie::Extensions::DeepLocate
end
end

def filtered_mashes
mashes.select { |mash| mash_matches_filter? mash }
end

def mash_matches_filter? mash

# `.all?` requires all filters to match, `.any?` requires only one

@filters.all? { |filter| filter.call mash }
end
end

# All that for this

task :get_products => :environment do
@all_products_from_all_categories = ProductsFromOtherStore.fetch

@all_products_from_all_categories.each do |products|
products.each do |product|
puts product.to_h
end
end
end


The messy JSON we got via rest-client:

{
"productHeader": {
"totalHits": 367
},
"products": [{
"name": "501 CT Jeans For Women",
"productImage": {
"url": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-1056.jpg"
},
"language": "no",
"description": "Jeans fra Levi's. Noe kortere nederst, fem lommer. Normal høyde på midjen, med hemper i linningen og knappegylfen. Dekorative slitte partier foran og nederst på benet.<br />Laget av 100% bomull.",
"brand": "Levis",
"identifiers": {
"sku": "441576-1056"
},
"fields": [{
"name": "sale",
"value": "false"
}, {
"name": "sizes",
"value": "W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W25/L34,W26/L34,W27/L34,W28/L34,W29/L34,W30/L34"
}, {
"name": "productStyle",
"value": "Straight"
}, {
"name": "gender",
"value": "Kvinne"
}, {
"name": "product_id_original",
"value": "441576-1056"
}, {
"name": "productGroup",
"value": "Jeans"
}, {
"name": "extraImageProductLarge",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-1056.jpg"
}, {
"name": "extraImageProductSmall",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/441576-1056.jpg"
}, {
"name": "artNumber",
"value": "441576-1056"
}, {
"name": "productClass",
"value": "Klær"
}, {
"name": "color",
"value": "Indigo"
}],
"offers": [{
"feedId": 10086,
"productUrl": "http://pdt.tradedoubler.com/click?a(2402331)p(80279)product(57d37b9ce4b085c06c38c96b)ttid(3)url(http%3A%2F%2Fnelly.com%2Fno%2Fkl%C3%A6r-til-kvinner%2Fkl%C3%A6r%2Fjeans%2Flevis-441%2F501-ct-jeans-for-women-441576-1056%2F)",
"priceHistory": [{
"price": {
"value": "1195",
"currency": "NOK"
},
"date": 1473477532181
}],
"modified": 1473477532181,
"inStock": 1,
"sourceProductId": "441576-1056",
"programLogo": "http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png",
"programName": "Nelly NO",
"id": "57d37b9ce4b085c06c38c96b"
}],
"categories": [{
"name": "Kvinne > KLÆR > Jeans > Straight"
}]
}, {
"name": "501 CT Jeans For Women",
"productImage": {
"url": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-6581.jpg"
},
"language": "no",
"description": "Jeans fra Levi's. Noe kortere nederst, fem lommer. Normal høyde på midjen, med hemper i linningen og knappegylfen. Dekorative slitte partier foran og nederst på benet.<br />Laget av 100% bomull.",
"brand": "Levis",
"identifiers": {
"sku": "441576-6581"
},
"fields": [{
"name": "sale",
"value": "false"
}, {
"name": "artNumber",
"value": "441576-6581"
}, {
"name": "productStyle",
"value": "Straight"
}, {
"name": "gender",
"value": "Kvinne"
}, {
"name": "extraImageProductLarge",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/productLarge/441576-6581.jpg"
}, {
"name": "extraImageProductSmall",
"value": "http://nlyscandinavia.scene7.com/is/image/nlyscandinavia/cart_thumb/441576-6581.jpg"
}, {
"name": "productGroup",
"value": "Jeans"
}, {
"name": "product_id_original",
"value": "441576-6581"
}, {
"name": "productClass",
"value": "Klær"
}, {
"name": "color",
"value": "Desert"
}, {
"name": "sizes",
"value": "W24/L32,W25/L32,W26/L32,W27/L32,W28/L32,W29/L32,W30/L32,W31/L32,W25/L34,W26/L34,W27/L34,W28/L34,W29/L34,W30/L34,W31/L34"
}],
"offers": [{
"feedId": 10086,
"productUrl": "http://pdt.tradedoubler.com/click?a(2402331)p(80279)product(57b3cafbe4b06cf59bc254bf)ttid(3)url(http%3A%2F%2Fnelly.com%2Fno%2Fkl%C3%A6r-til-kvinner%2Fkl%C3%A6r%2Fjeans%2Flevis-441%2F501-ct-jeans-for-women-441576-6581%2F)",
"priceHistory": [{
"price": {
"value": "1195",
"currency": "NOK"
},
"date": 1471400699283
}],
"modified": 1471400699283,
"inStock": 1,
"sourceProductId": "441576-6581",
"programLogo": "http://hst.tradedoubler.com/file/17833/2014-logos/200X200.png",
"programName": "Nelly NO",
"id": "57b3cafbe4b06cf59bc254bf"
}],
"categories": [{
"name": "Kvinne > KLÆR > Jeans > Straight"
}]
}]
}

Answer

There is a lot of things going on in your code sample. I tried to split in parts and restructure it. It does not do the same as your code but I think it should get you started and perhaps you can come back when you have a more specific question.

Note that I did not use hashie, I think that accessing some deeply nested hash structures in a few places does not justify adding a new library to a project.

Questions/Ideas/Hints:

  • are prices Integers or Floats?
  • Is the JSON consistent (all elements present all the time?)
  • Are you using Ruby 2.3? Then look into Hash#dig
  • Why did you prettify the JSON keys? Does not make sense to me as you build Product objects to work with anyway?
  • Unless there are performance issues i would convert all products to Ruby objects first and filter then. Just easier and more readable.

Code

Product (same as yours)

Product = Struct.new(:category, :name, :image, :price, :description)

JsonProductBuilder converts the parsed JSON to Product Objects.

class JsonProductBuilder
  def initialize(json)
    @json = json
  end

  def call
    json.fetch('products', []).map do |item|
      Product.new(
        extract_category(item),
        item['name'],
        item.fetch('productImage', {})['url'],
        extract_price(item),
        item['description']
      )
    end
  end

  private

  attr_reader :json

  def extract_category(item)
    field = item['fields'].find do |field|
      field['name'] == 'productGroup'
    end
    field['value'] if field
  end

  def extract_price(item)
    offer = item['offers'].first
    history = offer['priceHistory'].first
    value = history['price']['value']
    Integer(value) # Or use Float?
  end
end

CategoryFilter returns a limited subset of the products. You can easily add other filters and combine them. Perhaps you might want to look into lazy for performance improvements.

class CategoryFilter
  def initialize(products, *categories)
    @products = products
    @categories = categories
  end

  def call
    products.select do |product|
      categories.include?(product.category)
    end
  end

  private

  attr_reader :products, :categories
end

Use it like this:

response = RestClient.get(
  'http://api.tradedoubler.com/1.0/products.json',
  {params: {q: 'Jeans', limit: LIMIT, token: KEY}}
)

json = JSON.parse(response)

products = JsonProductBuilder.new(json).call
puts products.size

products = CategoryFilter.new(products, 'Klær', 'Sko', 'Jeans').call
puts products.size

products.each do |product|
  puts product.to_h
end