Web Scraping Weather Data Using Node JS and Puppeteer

mkdir puppeteercd puppeteernpm install puppeteer --save
const puppeteer = require('puppeteer');puppeteer.launch({ headless: true, args: ['--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3312.0 Safari/537.36"'] }).then(async browser => {	const page = await browser.newPage();
await page.goto("https://weather.com/en-IN/weather/tenday/l/aff9460b9160c73ff01769fd83ae82cf37cb27fb7eb73c70b91257d413147b69");
await page.waitForSelector('body');
var rposts = await page.evaluate(() => {
}); console.log(rposts);
await browser.close();
}).catch(function(error) {
console.error(error);
});
node get_weather.js
const puppeteer = require('puppeteer');puppeteer.launch({ headless: true, args: ['--user-agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3312.0 Safari/537.36"'] }).then(async browser => {	const page = await browser.newPage();
await page.goto("https://weather.com/en-IN/weather/tenday/l/892a73a36fbd270bd1d5230d6b8a00a7da793b50b9878f50b49360ba0e072dc5");
await page.waitForSelector('body');
var rposts = await page.evaluate(() => {
let posts = document.body.querySelectorAll('.clickable');
postItems = [];
posts.forEach((item) => { let dayDetail = ''
let description = ''
let temp = ''
let humidity = ''
let wind = ''
let precip = ''
try{
dayDetail = item.querySelector('.day-detail').innerText;
if (dayDetail!=''){
description = item.querySelector('.description').innerText;
temp = item.querySelector('.temp').innerText;
precip = item.querySelector('.precip').innerText;
humidity = item.querySelector('.humidity').innerText;
wind = item.querySelector('.wind').innerText;
postItems.push({dayDetail: dayDetail, description: description, temp : temp, precip : precip, humidity: humidity, wind: wind});
}
}catch(e){
} });

var items = {
"posts": postItems
};
return items;

});
console.log(rposts);
await browser.close();
}).catch(function(error) {
console.error(error);
});
  • With millions of high speed rotating proxies located all over the world.
  • With our automatic IP rotation.
  • With our automatic User-Agent-String rotation (which simulates requests from different, valid web browsers and web browser versions).
  • With our automatic CAPTCHA solving technology.

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store