Scraping Job Listings with Python and BeautifulSoup

# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9'}
url = 'https://www.indeed.com/jobs?q=web crawler'
response=requests.get(url,headers=headers)#print(response.content)
soup=BeautifulSoup(response.content,'lxml')
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9'}
url = 'https://www.indeed.com/jobs?q=web crawler'
response=requests.get(url,headers=headers)#print(response.content)
soup=BeautifulSoup(response.content,'lxml')
for item in soup.select('.jobsearch-SerpJobCard'):
try:
print('----------------------------------------')
print(item)
except Exception as e:
#raise e
print('')
python3 scrapeIndeed.py
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9'}
url = 'https://www.indeed.com/jobs?q=web crawler'
response=requests.get(url,headers=headers)#print(response.content)
soup=BeautifulSoup(response.content,'lxml')
for item in soup.select('.jobsearch-SerpJobCard'):
try:
print('----------------------------------------')
print(item.select('.title')[0].get_text().strip()) print('----------------------------------------')
except Exception as e:
#raise e
print('')
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/601.3.9 (KHTML, like Gecko) Version/9.0.2 Safari/601.3.9'}
url = 'https://www.indeed.com/jobs?q=web crawler'
response=requests.get(url,headers=headers)#print(response.content)
soup=BeautifulSoup(response.content,'lxml')
for item in soup.select('.jobsearch-SerpJobCard'):
try:
print('----------------------------------------')

print(item.select('.title')[0].get_text().strip())
print(item.select('.company')[0].get_text().strip())
print(item.select('.location')[0].get_text().strip())
print(item.select('.summary')[0].get_text().strip())
print('----------------------------------------')
except Exception as e:
#raise e
print('')
  • With millions of high speed rotating proxies located all over the world,
  • With our automatic IP rotation
  • With our automatic User-Agent-String rotation (which simulates requests from different, valid web browsers and web browser versions)
  • With our automatic CAPTCHA solving technology,
curl "http://api.proxiesapi.com/?key=API_KEY&url=https://example.com"

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store