nuccio nuccio -4 years ago 115
HTML Question

Error Code: 402 when trying to capture Web Page with Casperjs

I can't capture a web page properly that I am logging into with Casperjs. I get a picture of a Web Page that simply has an H1 tag with "Not Implemented". Here is the script below:

var casper = require('casper').create({
pageSettings: {
loadImages: false,//The script is much faster when this field is set to false
loadPlugins: false,
userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
customHeaders:{
'Authorization':'Basic '+btoa('someusername:somepassword')
}
}
});

casper.on("resource.error", function(resourceError){
console.log('Unable to load resource (#' + resourceError.id + 'URL:' + resourceError.url + ')');
console.log('Error code: ' + resourceError.errorCode + '. Description: ' + resourceError.errorString);
});


//First step is to open SkySpark Login
casper.start().thenOpen("http://192.168.9.150:89/user/login", function() {
console.log("SkySpark website opened");
});

//Second step is to click to the Sign-in button
casper.then(function(){
this.evaluate(function(){
document.getElementById("nav-tools").children[0].click();
});
});

//Now we have to populate username and password, and submit the form
casper.then(function(){
console.log("Login using username and password");
this.evaluate(function(){
document.getElementById("username").value="someusername";
document.getElementById("password").value="somepassword";
document.getElementById("loginForm").submit();
});
});

//Wait to be redirected to the Home page, and then make a screenshot
casper.then(function(){
console.log("Make a screenshot and save it as SkySparkTens.png");
this.capture('SkySparkTens.png');
});

//prints HTML to the console
casper.then(function(){
this.wait(5000, function() {
console.log(this.getHTML() );
});
});

casper.run();


Here is the console response I get back:

D:\temp\CasperLogin>casperjs logCasper.js

SkySpark website opened

Login using username and password

Unable to load resource (#4URL:http://192.168.9.150:89/user/login)
Error code: 402. Description: Error downloading http://192.168.9.150:89/user/log
in - server replied: Not Implemented

Make a screenshot and save it as SkySparkTens.png

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>501 Not Implemented</title>
</head>
<body>
<h1>Not Implemented</h1>
</body>
</html>


Here is the form that appears on the login page to the site that I am trying to access.

<script type='text/javascript'>
userModLogin.passwordRequired = false;
userModLogin.authUri = "/user/auth";
userModLogin.redirectUri = '/ui/';
userModLogin.localeLogin = 'Login';
userModLogin.localeLoggingIn = 'Logging in';
userModLogin.localeBadCres = 'Invalid username or password';
userModLogin.autoFocusId = 'username';
window.onload = function() { userModLogin.init(false); }
</script>
</head>
<body>
<form id='loginForm' method='post' action='/user/login'>
<p class='logo'>
<img src='/brand/logo.svg' title='SkySpark' alt='SkySpark' /> </p>
<p id='err'>
Invalid username or password</p>
<p>
<label for='username'>
Username:</label>
<input type='text' id='username' name='username'placeholder='Username' /></p>
<p>
<label for='password'>Password:</label>
<input type='password' id='password' name='password' size='25' autocomplete='off' placeholder='Password' /></p>
<p>
<label for='mobile'>
<input type='checkbox' id='mobile' value='mobile' /> Mobile</label>
</p>
<p>
<input type='submit' id='loginButton' value='Login' onclick='return userModLogin.loginAuth();' /></p>
</form>
</body>

Answer Source

I finally have gotten my script to work properly, also I made the script more dynamic than the original script that I had posted.

I realized that I had two issues that was stopping my script from working. The first issue is that the capture was not waiting for the page to load completely before taking creating the pdf. This line below gets the URL and waits for it to load. This ensures that the screen capture later in the script creates a pdf of a fully loaded page.

  this.waitForUrl(casper.cli.get("report"), function() {
    casper.back();
  });   

I also increased the timeout time from 5 seconds to 25 seconds with this line below to make sure data heavy reports are loaded without timing out.

 casper.options.waitTimeout = 25000; 

The second issue was that the Webpage I was trying to create a PDF from uses a Fantom pod called Domkit. Domkit could not be loaded properly in the Phantomjs Virtual Browser. I resolved this issue by removing domkit from the UI of the webpage I was trying to capture with Casperjs.

Here is the full script below

 /*
 note: all parameters should be formatted like so on the command line: --  parameter=value 
 note: all paramters must be separated by a space from each other: -- paramter1=value1 parameter2=value2
 Parameter 1: --filename (required, Whatever name given here will be the name of the file)
 Parameter 2: --username (required, This needs to be the users SkySpark username)
 Parameter 3: --password (required, This needs to be the users SkySPark password)
 Parameter 4: --report (required, url needed to get to the desired report in SkySpark)
 Parameter 4: --directory (optional will default to the io directory if null. This is the path needed to 
                      get to the desired place to store file. All \ must be entered as /)  
 Parameter 5: --fileType (optional will default to PDF if null, fileType needed to render the file as desired type(pdf,png,jpeg,jpg), 
                      only png and pdf are supported)    
 */
 var casper = require('casper').create({
   // verbose: true, 
   // logLevel: 'debug',
   pageSettings: {
     loadImages: true,//The script is much faster when this field is set to false
     loadPlugins: false,
     userAgent: 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36'
   }
 });

 //First step is to open the report in SkySpark.
 casper.start().thenOpen(casper.cli.get("report"), function() {
    console.log("SkySpark website opened");
 });

 //Changes the timeout from 5 seconds to 25 seconds
 casper.options.waitTimeout = 25000; 


 //Now we have to populate username and password, and submit the form
 casper.then(function(username, password){
     console.log("Login using username and password");
//take user name and password from variables and creates varaible to pass into the evaluate
var uName = casper.cli.get("username");
var pWord = casper.cli.get("password")
/*function that will be executed in the DOM context (you can also call it the page context). 
You can pass some primitives as arguments to this function and return one primitive back. 
Keep in mind that this function that you pass to evaluate must be self contained. It cannot use 
variables or functions that are defined outside of this function.
*/
this.evaluate(function(username, password){
    //SkySpark
    document.getElementById("username").value = username;
    document.getElementById("password").value = password;
    document.getElementById("loginButton").click();
}, uName, pWord);
//This waits for the report to load completely before continueing
this.waitForUrl(casper.cli.get("report"), function() {
    casper.back();
});

});
//Gets the path from the parameters, if user puts null as parameter the paramter will be set to the io directory in SkySpark
var directory;
if(casper.cli.get("directory") == null || casper.cli.get("directory").length <= 1){
   directory = 'D:\\SkySpark\\skyspark-3.0.9\\var\\proj\\development\\io\\';
}else{
   directory = casper.cli.get("directory");
}
//fileType determines what the captured file will be.
var fileType;
if(casper.cli.get("fileType") == "png"){
    fileType = '.png';
}else if(casper.cli.get("fileType") == "jpg" || casper.cli.get("fileType") =="jpeg" ){
    fileType = '.jpeg';
}else{
    fileType = '.pdf';
}

//Wait to be redirected to the Home page, and then make a screenshot
casper.then(function(){
    console.log("Make a screenshot and save it as Sky Home Page");
    this.capture(directory + casper.cli.get("filename") + fileType);
    console.log(directory + casper.cli.get("filename") + fileType);
    this.echo(this.getCurrentUrl());
});

casper.run();
Recommended from our users: Dynamic Network Monitoring from WhatsUp Gold from IPSwitch. Free Download