Add files via upload

Main file for the project. external file named urlname.py needed with url variable set to the url of the site you want to scrape
main
Crimson Tome 2021-05-19 20:05:30 +01:00 committed by GitHub
commit 4dea39acf0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 31 additions and 0 deletions

31
webScraper.py Normal file
View File

@ -0,0 +1,31 @@
import requests
from bs4 import BeautifulSoup
from time import sleep
import urlname
def getdata(url):
r = requests.get(url)
return r.text
htmldata = getdata(urlname.url)
soupData = BeautifulSoup(htmldata, 'html.parser')
listofimages = []
count = 1
for item in soupData.find_all('img'):
listofimages.append(item['src'])
for i in listofimages:
try:
print (i)
response = requests.get(i)
file = open("image"+str(count)+".png", "wb")
file.write(response.content)
file.close()
count+=1
print("download of "+i+" complete \n")
sleep(1)
except:
print("Sorry, something went wrong. Moving onto the next image \n")