In [94]:
import pandas as pd
from bs4 import BeautifulSoup
import requests
import json
import urllib
In [71]:
url='http://quotes.toscrape.com/'
response=requests.get(url)
response.status_code
soup=BeautifulSoup(response.content,'html.parser')
quote=soup.find('span',class_="text")
print(quote)

<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>

quote=soup.find('span',class_="text")

In [76]:
quotes=soup.find_all('span',class_="text")
print(quotes)

[<span class="text" itemprop="text">“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”</span>, <span class="text" itemprop="text">“It is our choices, Harry, that show what we truly are, far more than our abilities.”</span>, <span class="text" itemprop="text">“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”</span>, <span class="text" itemprop="text">“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”</span>, <span class="text" itemprop="text">“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”</span>, <span class="text" itemprop="text">“Try not to become a man of success. Rather become a man of value.”</span>, <span class="text" itemprop="text">“It is better to be hated for what you are than to be loved for what you are not.”</span>, <span class="text" itemprop="text">“I have not failed. I've just found 10,000 ways that won't work.”</span>, <span class="text" itemprop="text">“A woman is like a tea bag; you never know how strong it is until it's in hot water.”</span>, <span class="text" itemprop="text">“A day without sunshine is like, you know, night.”</span>]
In [77]:
quotes=[quote.text[1:-1] for quote in quotes]
quotes
Out[77]:
['The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.',
 'It is our choices, Harry, that show what we truly are, far more than our abilities.',
 'There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.',
 'The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.',
 "Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.",
 'Try not to become a man of success. Rather become a man of value.',
 'It is better to be hated for what you are than to be loved for what you are not.',
 "I have not failed. I've just found 10,000 ways that won't work.",
 "A woman is like a tea bag; you never know how strong it is until it's in hot water.",
 'A day without sunshine is like, you know, night.']
In [80]:
authors=soup.find_all('small',class_="author")
authors
Out[80]:
[<small class="author" itemprop="author">Albert Einstein</small>,
 <small class="author" itemprop="author">J.K. Rowling</small>,
 <small class="author" itemprop="author">Albert Einstein</small>,
 <small class="author" itemprop="author">Jane Austen</small>,
 <small class="author" itemprop="author">Marilyn Monroe</small>,
 <small class="author" itemprop="author">Albert Einstein</small>,
 <small class="author" itemprop="author">André Gide</small>,
 <small class="author" itemprop="author">Thomas A. Edison</small>,
 <small class="author" itemprop="author">Eleanor Roosevelt</small>,
 <small class="author" itemprop="author">Steve Martin</small>]
In [81]:
authors=[author.text for author in authors]
authors
Out[81]:
['Albert Einstein',
 'J.K. Rowling',
 'Albert Einstein',
 'Jane Austen',
 'Marilyn Monroe',
 'Albert Einstein',
 'André Gide',
 'Thomas A. Edison',
 'Eleanor Roosevelt',
 'Steve Martin']
In [82]:
tags=soup.find_all('div',class_="tags")
In [83]:
tags[0]
Out[83]:
<div class="tags">
            Tags:
            <meta class="keywords" content="change,deep-thoughts,thinking,world" itemprop="keywords"/>
<a class="tag" href="/tag/change/page/1/">change</a>
<a class="tag" href="/tag/deep-thoughts/page/1/">deep-thoughts</a>
<a class="tag" href="/tag/thinking/page/1/">thinking</a>
<a class="tag" href="/tag/world/page/1/">world</a>
</div>
In [84]:
for i in tags[0].find_all('a',class_='tag'):
    print(i.text)

change
deep-thoughts
thinking
world
In [87]:
total_tags=[]
for i in range(len(tags)):
    k=[]
    for j in tags[i].find_all('a',class_='tag'):
        k.append(j.text)
    total_tags.append(','.join(k))
total_tags
Out[87]:
['change,deep-thoughts,thinking,world',
 'abilities,choices',
 'inspirational,life,live,miracle,miracles',
 'aliteracy,books,classic,humor',
 'be-yourself,inspirational',
 'adulthood,success,value',
 'life,love',
 'edison,failure,inspirational,paraphrased',
 'misattributed-eleanor-roosevelt',
 'humor,obvious,simile']
In [88]:
dataset=pd.DataFrame()
dataset['Quote']=quotes
dataset['Tags']=total_tags
dataset['Authors']=authors
dataset
Out[88]:
Quote Tags Authors
0 The world as we have created it is a process o... change,deep-thoughts,thinking,world Albert Einstein
1 It is our choices, Harry, that show what we tr... abilities,choices J.K. Rowling
2 There are only two ways to live your life. One... inspirational,life,live,miracle,miracles Albert Einstein
3 The person, be it gentleman or lady, who has n... aliteracy,books,classic,humor Jane Austen
4 Imperfection is beauty, madness is genius and ... be-yourself,inspirational Marilyn Monroe
5 Try not to become a man of success. Rather bec... adulthood,success,value Albert Einstein
6 It is better to be hated for what you are than... life,love André Gide
7 I have not failed. I've just found 10,000 ways... edison,failure,inspirational,paraphrased Thomas A. Edison
8 A woman is like a tea bag; you never know how ... misattributed-eleanor-roosevelt Eleanor Roosevelt
9 A day without sunshine is like, you know, night. humor,obvious,simile Steve Martin
In [113]:
d=df.to_csv('quotes.csv')
In [ ]: