Aziz Aziz - 3 months ago 7
Javascript Question

Store multiple scraping data in an Array

First part:

Currently I scrapes a webpage to get links, then I open each scraped link this part work perfectly.

second part:

I check if there is selection of a (select) options field and stored the values(id) in an array this part also works fine.

Third part:

I want to loop through them(selection of options) and fire a click event and wait for AJAX response then extracts infoProduct then stor it in an array.

I have trouble in this part I got an empty array because the

return listProducts
is called before the first
this.eachThen
start.

function getInfosProduct(obj,variation) {
if (!variation) {
return [{
name: getName(),
image: products.image.getElement.link + getImage(),
url: obj.url
}];
} else {
return {
name: getName(),
image: products.image.getElement.link + getImage(),
url: variation.url,
idVariation: variation.id,
descVariation: variation.description
};
}
}

function clickVariation(variation) {
if (variation.ok && variation.id != variation.ignore) {
chooseVariation(products.selector, variation.id);
return true;
}
return false;
}

casper.getInfosProducts = function(obj) {
if (obj.level == 0) {
return this.evaluate(getInfosProduct, obj,false);
} else {
listProducts = [];
this.eachThen(obj.levelVariation, function getInfosProducts(variation) {
isClick = this.evaluate(clickVariation, variation.data)
if (isClick) {

this.waitForSelectorTextChange('.selector', function() {
this.echo('The text on .selector has been changed.');
});

listProducts.push(this.evaluate(getInfosProduct, obj,variation.data));
}
});
return listProducts;
}
};


Function to trigger the change event on the select element

function chooseVariation(selector, valueToMatch) {
var select = document.querySelectorAll(selector),
found = false;
Array.prototype.forEach.call(select, function(opt, i) {
if (!found && opt.value.indexOf(valueToMatch) !== -1) {
select.selectedIndex = i;
found = true;
}
});
// dispatch change event in case there is some kind of validation
var evt = document.createEvent("UIEvents"); // or "HTMLEvents"
evt.initUIEvent("change", true, true);
select[0].dispatchEvent(evt);
}


This is the main fucntion:

function startSraping(obj) {
casper.then(function switchAction() {
switch (obj.action) {

//......... some code ........

// iterating through array links and open pages
case "openLinks":
this.each(links, function eachOpenLinks(self, link) {
if (link.ok) {
self.thenOpen(link.url, function thenOpenLinks() {
startSraping({
url: link.url,
action: "getVariations"
});
});
}
});
break;

// get all variations for each page opend
case "getVariations":
objVariations = this.getVariations(obj.url);
startSraping({
url: obj.url,
action: "getInfosProducts",
objVariations: objVariations
});
break;

case "getInfosProducts":
this.eachThen(obj.objVariations.list, function(levelVariation) {
infosProd = this.getInfosProducts({
levelVariation: levelVariation.data,
url: obj.url,
level: obj.objVariations.level
});
// Here I got an empty array
this.echo(JSON.stringify(infosProd), 'INFO');
});
break;
}
});
}
casper.start(url, function start() {
startSraping({
variation: variation,
action: "submitSearch"
});
});
casper.run();

Answer

You can't call an asynchronous function (eachThen and waitForSelectorTextChange are both asynchronous) inside of a function that is supposed to return the result from the asynchronous function in a synchronous fashion (general reference). Since CasperJS doesn't support Promises, this get's a little tricky.

I think the following changes should be minimal and get you where you want to go.

casper.getInfosProducts = function(obj, callback) {
    if (obj.level == 0) {
        this.then(function(){
            callback.call(this, arr.push(this.evaluate(getInfosProduct, obj,false));
        });
    } else {
        var listProducts = [];
        this.eachThen(obj.levelVariation, function getInfosProducts(variation) {
            var isClick = this.evaluate(clickVariation, variation.data)
            if (isClick) {
                this.waitForSelectorTextChange('.selector', function() {
                    this.echo('The text on .selector has been changed.');
                    listProducts.push(this.evaluate(getInfosProduct, obj, variation.data));
                });
            }
        });
        this.then(function(){
            callback.call(this, listProducts);
        });
    }
};

In startSraping:

case "getInfosProducts":
    this.eachThen(obj.objVariations.list, function(levelVariation) {
        this.getInfosProducts({
            levelVariation: levelVariation.data,
            url: obj.url,
            level: obj.objVariations.level
        }, function (infosProd){
            // this is the asynchronous callback
            this.echo(JSON.stringify(infosProd), 'INFO');
        });
    });
    break;