Harish Sundaresan Harish Sundaresan - 4 months ago 20
Ajax Question

I used the website's URL before using Ajax URL which parsed the table. Now I get "none" when i print

import requests

import csv

from BeautifulSoup import BeautifulSoup

payload = {
'action': "historical_data",
'curr_id': "160",
'st_date' : "01/05/2016",
'end_date' : "30/05/2016",
'interval_sec' : "Daily"
}

headers = {
'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
'Content-Type': 'application/x-www-form-urlencoded'
}

url = 'http://in.investing.com/instruments/HistoricalDataAjax'

response=requests.post(url,data=payload,headers=headers)

html = response.content

soup = BeautifulSoup(html)

print soup.find('table', attrs={'class':'genTbl closedTbl historicalTbl'})

Answer

First off use bs4, BeautifulSoup3 is deprecated and no longer maintained, to fix your issue you need to add another header, "X-Requested-With":"XMLHttpRequest":

from bs4 import BeautifulSoup
import requests


payload = {
    'action': "historical_data",
    'curr_id': "160",
    'st_date' : "01/05/2016",
    'end_date' : "30/05/2016",
    'interval_sec' : "Daily"
}
headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:46.0) Gecko/20100101 Firefox/46.0',
        'Content-Type': 'application/x-www-form-urlencoded',
    "X-Requested-With":"XMLHttpRequest"
}
url = 'http://in.investing.com/instruments/HistoricalDataAjax'
response=requests.post(url, data=payload,headers=headers)
html = response.content
soup = BeautifulSoup(html,"html.parser")
print(soup.find('table', attrs={'class':'genTbl closedTbl historicalTbl'}))

Output:

<table class="genTbl closedTbl historicalTbl" id="curr_table" tablesorter="">
<thead>
<tr>
<th class="first left noWrap">Date<span class="headerSortDefault" sort_default=""></span></th>
<th class="noWrap">Price<span class="headerSortDefault" sort_default=""></span></th>
<th class="noWrap">Open<span class="headerSortDefault" sort_default=""></span></th>
<th class="noWrap">High<span class="headerSortDefault" sort_default=""></span></th>
<th class="noWrap">Low<span class="headerSortDefault" sort_default=""></span></th>
<th class="noWrap">Change %<span class="headerSortDefault" sort_default=""></span></th>
</tr>
</thead>
<tbody>
<tr>
<td class="first left bold noWrap" data-real-value="1464566400">May 30, 2016</td>
<td class="greenFont">67.171</td>
<td>67.030</td>
<td>67.380</td>
<td>67.022</td>
<td class="bold greenFont">0.21%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1464307200">May 27, 2016</td>
<td class="greenFont">67.031</td>
<td>66.932</td>
<td>67.135</td>
<td>66.865</td>
<td class="bold greenFont">0.16%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1464220800">May 26, 2016</td>
<td class="redFont">66.927</td>
<td>67.290</td>
<td>67.415</td>
<td>66.892</td>
<td class="bold redFont">-0.51%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1464134400">May 25, 2016</td>
<td class="redFont">67.273</td>
<td>67.620</td>
<td>67.712</td>
<td>67.213</td>
<td class="bold redFont">-0.53%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1464048000">May 24, 2016</td>
<td class="greenFont">67.630</td>
<td>67.419</td>
<td>67.780</td>
<td>67.407</td>
<td class="bold greenFont">0.33%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463961600">May 23, 2016</td>
<td class="greenFont">67.409</td>
<td>67.421</td>
<td>67.590</td>
<td>67.233</td>
<td class="bold greenFont">0.00%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463702400">May 20, 2016</td>
<td class="redFont">67.408</td>
<td>67.426</td>
<td>67.545</td>
<td>67.310</td>
<td class="bold redFont">-0.04%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463616000">May 19, 2016</td>
<td class="greenFont">67.436</td>
<td>67.133</td>
<td>67.610</td>
<td>67.093</td>
<td class="bold greenFont">0.44%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463529600">May 18, 2016</td>
<td class="greenFont">67.140</td>
<td>66.814</td>
<td>67.169</td>
<td>66.814</td>
<td class="bold greenFont">0.47%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463443200">May 17, 2016</td>
<td class="redFont">66.825</td>
<td>66.835</td>
<td>66.901</td>
<td>66.665</td>
<td class="bold redFont">-0.03%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463356800">May 16, 2016</td>
<td class="redFont">66.844</td>
<td>66.915</td>
<td>66.944</td>
<td>66.720</td>
<td class="bold redFont">-0.02%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463097600">May 13, 2016</td>
<td class="greenFont">66.855</td>
<td>66.748</td>
<td>66.925</td>
<td>66.710</td>
<td class="bold greenFont">0.15%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1463011200">May 12, 2016</td>
<td class="greenFont">66.757</td>
<td>66.551</td>
<td>66.781</td>
<td>66.510</td>
<td class="bold greenFont">0.29%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462924800">May 11, 2016</td>
<td class="redFont">66.561</td>
<td>66.621</td>
<td>66.845</td>
<td>66.510</td>
<td class="bold redFont">-0.11%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462838400">May 10, 2016</td>
<td class="redFont">66.631</td>
<td>66.742</td>
<td>66.786</td>
<td>66.532</td>
<td class="bold redFont">-0.18%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462752000">May 09, 2016</td>
<td class="greenFont">66.752</td>
<td>66.591</td>
<td>66.800</td>
<td>66.385</td>
<td class="bold greenFont">0.23%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462492800">May 06, 2016</td>
<td class="greenFont">66.600</td>
<td>66.558</td>
<td>66.715</td>
<td>66.470</td>
<td class="bold greenFont">0.06%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462406400">May 05, 2016</td>
<td class="redFont">66.558</td>
<td>66.617</td>
<td>66.675</td>
<td>66.448</td>
<td class="bold redFont">-0.10%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462320000">May 04, 2016</td>
<td class="greenFont">66.627</td>
<td>66.569</td>
<td>66.689</td>
<td>66.490</td>
<td class="bold greenFont">0.10%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462233600">May 03, 2016</td>
<td class="greenFont">66.558</td>
<td>66.333</td>
<td>66.582</td>
<td>66.225</td>
<td class="bold greenFont">0.32%</td>
</tr>
<tr>
<td class="first left bold noWrap" data-real-value="1462147200">May 02, 2016</td>
<td class="redFont">66.347</td>
<td>66.415</td>
<td>66.465</td>
<td>66.310</td>
<td class="bold redFont">-0.12%</td>
</tr>
</tbody>
</table>
Comments