In [ ]:
import requests
from bs4 import BeautifulSoup

url = "https://www.google.co.in/"

r = requests.get(url)
htmlcontent = r.content
#print(htmlcontent)

soup = BeautifulSoup(htmlcontent, 'html.parser')
print(soup.prettify)

title = soup.title
#print(type(title))
paras = soup.find_all('a')
#print(paras)

anchors = soup.find_all('a')
#print(anchors)

print(soup.find('p'))

print(soup.find('p')['class'])

print(soup.find_all("p", class_="lead"))

print(soup.find('P').get_text())

print(soup.get_text())
In [ ]:
import requests


r = requests.get('https://handbook.mattermost.com/')

print(r)



print(r.content)
In [2]:
r = requests.get('https://handbook.mattermost.com/')


print(r.url)

print(r.status_code)

https://handbook.mattermost.com/
200
In [4]:
import requests
from bs4 import BeautifulSoup


r = requests.get('https://handbook.mattermost.com/')

soup = BeautifulSoup(r.content, 'html.parser')

print(soup.title)

print(soup.title.name)

print(soup.title.parent.name)

<title>Mattermost Handbook | 0.2.1 | Mattermost Handbook</title>
title
head
In [17]:
print(soup.find_all("p", class_="lead"))

[]
In [9]:
for link in soup.find_all('a'):
    print(link.get('href'))

/
https://handbook.mattermost.com/v/0.2.0/
https://handbook.mattermost.com/
/help-and-support/contact-us
https://handbook.mattermost.com/operations/finance/staff-member-expenses/how-to-spend-company-money
https://handbook.mattermost.com/operations/finance/staff-member-expenses/how-to-use-expensify
https://handbook.mattermost.com/company/how-to-guides-for-staff/how-to-update-handbook
https://handbook.mattermost.com/operations/research-and-development/product/release-process/release-overview
/
/company/about-mattermost
/company/about-mattermost/list-of-terms
/company/about-mattermost/business-model
/company/about-mattermost/mindsets
/company/how-to-guides-for-staff
/company/how-to-guides-for-staff/how-to-setup-a-1-1-channel
/company/how-to-guides-for-staff/how-to-update-handbook
/company/how-to-guides-for-staff/how-to-manage-handbook-notifications
/company/how-to-guides-for-staff/how-to-change-mobile-device
/company/how-to-guides-for-staff/how-to-change-mobile-device/how-to-handle-a-lost-mobile-device
/company/how-to-guides-for-staff/how-to-do-a-post-escalation-post-mortem
/company/how-to-guides-for-staff/how-to-autolink-keywords-in-mattermost
/operations/operations
/operations/operations/areas-of-responsibility
/operations/operations/mlt-cadence
/operations/operations/mlt-cadence/mlt-cadence-v2mom
/operations/operations/company-metrics
/operations/operations/company-metrics/metrics-definitions
/operations/operations/company-metrics/company-v2mom-metrics
/operations/operations/company-metrics/metrics
/operations/operations/company-cadence
/operations/operations/company-policies
/operations/operations/company-policies/community-response-policy
/operations/operations/company-policies/security-policies
/operations/operations/company-processes
/operations/operations/company-processes/issue-solution
/operations/operations/company-processes/company-agreements
/operations/operations/company-processes/publishing
/operations/operations/company-processes/publishing/web-properties
/operations/operations/company-processes/publishing/publishing-guidelines
/operations/operations/company-processes/publishing/publishing-guidelines/brand-and-visual-design-guidelines
/operations/operations/company-processes/publishing/publishing-guidelines/voice-tone-and-writing-style-guidelines
/operations/operations/company-processes/publishing/publishing-guidelines/voice-tone-and-writing-style-guidelines/contribute-to-documentation
/operations/operations/company-processes/publishing/publishing-guidelines/confidentiality-guidelines
/operations/operations/company-processes/publishing/quality-control-process
/operations/operations/handbook
/operations/operations/handbook/handbook-onboarding
/operations/operations/fiscal-year-planning
/operations/research-and-development
/operations/research-and-development/organization
/operations/research-and-development/organization/tech_writing
/operations/research-and-development/organization/data_engineering
/operations/research-and-development/organization/delivery
/operations/research-and-development/organization/cloud_platform
/operations/research-and-development/organization/sre
/operations/research-and-development/organization/grc
/operations/research-and-development/organization/product_security
/operations/research-and-development/organization/security_operations
/operations/research-and-development/product
/operations/research-and-development/product/product-planning
/operations/research-and-development/product/product-planning/product-philosophy-and-principles
/operations/research-and-development/product/product-planning/prioritization-process
/operations/research-and-development/product/product-planning/release-planning-process
/operations/research-and-development/product/product-planning/roadmap-views
/operations/research-and-development/product/product-planning/release-plan
/operations/research-and-development/product/product-planning/launch-plan
/operations/research-and-development/product/product-planning/feature-requests
/operations/research-and-development/product/development-process
/operations/research-and-development/product/development-process/mobile-feature-guidelines
/operations/research-and-development/product/development-process/deprecated-features
/operations/research-and-development/product/development-process/software-requirements
/operations/research-and-development/product/development-process/jira-ticket-lifecycle
/operations/research-and-development/product/development-process/new-bug-tickets
/operations/research-and-development/product/development-process/new-bug-tickets/bug-severity-guidelines
/operations/research-and-development/product/development-process/new-bug-tickets/jira-fix-versions
/operations/research-and-development/product/release-process
/operations/research-and-development/product/release-process/release-overview
/operations/research-and-development/product/release-process/feature-release
/operations/research-and-development/product/release-process/dot-release
/operations/research-and-development/product/release-process/security-release
/operations/research-and-development/product/release-process/mobile-release
/operations/research-and-development/product/release-process/desktop-release
/operations/research-and-development/product/release-process/release-tips
/operations/research-and-development/product/release-process/release-scorecard-definitions
/operations/research-and-development/product/how-to-guides-for-product
/operations/research-and-development/product/how-to-guides-for-product/how-to-use-productboard
/operations/research-and-development/product/how-to-guides-for-product/how-to-record-a-roadmap-video
/operations/research-and-development/product/how-to-guides-for-product/integrations-directory
/operations/research-and-development/product/how-to-guides-for-product/release-announcement
/operations/research-and-development/product/product-management-team-handbook
/operations/research-and-development/product/product-management-team-handbook/product-ownership-areas
/operations/research-and-development/product/product-management-team-handbook/pm-onboarding
/operations/research-and-development/product/product-management-team-handbook/pm-levels
/operations/research-and-development/product/product-management-team-handbook/professional-development
/operations/research-and-development/product/product-design-team-handbook
/operations/research-and-development/product/product-design-team-handbook/product-design-levels
/operations/research-and-development/product/technical-writing-team-handbook
/operations/research-and-development/product/technical-writing-team-handbook/work-with-us
/operations/research-and-development/product/technical-writing-team-handbook/user-interface-text-guidelines
/operations/research-and-development/product/technical-writing-team-handbook/documentation-style-guide
/operations/research-and-development/product/technical-writing-team-handbook/our-terminology
/operations/research-and-development/product/technical-writing-team-handbook/writing-feature-documentation
/operations/research-and-development/product/technical-writing-team-handbook/writing-community-documentation
/operations/research-and-development/product/technical-writing-team-handbook/writer-levels
/operations/research-and-development/product/technical-writing-team-handbook/docathon-2021
/operations/research-and-development/product/technical-writing-team-handbook/docathon-2021/docathon-processes
/operations/research-and-development/product/growth
/operations/research-and-development/product/growth/ab-testing
/operations/research-and-development/product/growth/pql
/operations/research-and-development/product/analytics
/operations/research-and-development/product/analytics/analyst-levels
/operations/research-and-development/product/analytics/looker
/operations/research-and-development/product/analytics/looker/dashboards
/operations/research-and-development/product/analytics/looker/explores
/operations/research-and-development/product/analytics/telemetry
/operations/research-and-development/product/developer-relations
/operations/research-and-development/product/product-team-hangouts
/operations/research-and-development/engineering
/operations/research-and-development/engineering/infrastructure-engineering
/operations/research-and-development/engineering/infrastructure-engineering/cloud-infrastructure-cost-kpis
/operations/research-and-development/engineering/infrastructure-engineering/cloud-data-export-process
/operations/research-and-development/engineering/infrastructure-engineering/cloud-churn-process
/operations/research-and-development/engineering/infrastructure-engineering/reliability-manifesto
/operations/research-and-development/engineering/infrastructure-engineering/production-readiness-review
/operations/research-and-development/engineering/infrastructure-engineering/infrastructure-library
/operations/research-and-development/engineering/integrations
/operations/research-and-development/engineering/plugin-release-process
/operations/research-and-development/engineering/data-engineering
/operations/research-and-development/engineering/team
/operations/research-and-development/engineering/team/on-call
/operations/research-and-development/engineering/how-to-go-to-conference
/operations/research-and-development/engineering/core-contributor-expanded-access
/operations/research-and-development/quality
/operations/research-and-development/quality/qa-workflow
/operations/research-and-development/quality/testing-tips-and-tools
/operations/research-and-development/quality/rainforest-process
/operations/messaging-and-math
/operations/messaging-and-math/how-to-guides-for-m-and-m
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-create-release-announcements
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-create-screenshots-and-gifs
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-write-mattermost-case-studies
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-write-guest-blog-posts-for-mattermost-apps-and-services
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-write-mattermost-recipes
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-compose-tweets
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-create-a-split-test-for-web-page
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-run-meetups
/operations/messaging-and-math/how-to-guides-for-m-and-m/how-to-run-executive-dinners
/operations/messaging-and-math/checklists-for-m-and-m
/operations/messaging-and-math/checklists-for-m-and-m/blog-post-checklist
/operations/messaging-and-math/checklists-for-m-and-m/bio-checklist
/operations/messaging-and-math/mattermost-websites
/operations/messaging-and-math/demand-generation-reporting
/operations/messaging-and-math/asana-guidelines
/operations/messaging-and-math/content-marketing
/operations/messaging-and-math/content-marketing/editorial-calendar
/operations/messaging-and-math/content-marketing/content-development-distribution-processes
/operations/messaging-and-math/content-marketing/video-content
/operations/messaging-and-math/content-marketing/contributing-content
/operations/sales
/operations/sales/deal-desk
/operations/sales/partner-programs
/operations/sales/lead-management
/operations/deployment-engineering
/operations/deployment-engineering/overview
/operations/deployment-engineering/workflows
/operations/deployment-engineering/frequently-asked-questions
/operations/deployment-engineering/playbook-mme-sev1-outages
/operations/deployment-engineering/status-update-template
/operations/program-management
/operations/customer-success
/operations/customer-success/customer-support
/operations/legal
/operations/legal/contracts
/operations/legal/ironclad-basics
/operations/legal/ironclad-basics/company-wide-templates
/operations/legal/ironclad-basics/requesting-sales-contracts
/operations/legal/ironclad-basics/signing-contracts
/operations/finance
/operations/finance/budget
/operations/finance/airbase
/operations/finance/airbase/how-to-access-airbase
/operations/finance/airbase/navigate-airbase
/operations/finance/airbase/how-to-submit-a-purchase-request
/operations/finance/airbase/how-to-submit-a-reimbursement-request
/operations/finance/airbase/how-to-review-a-reimbursement-request
/operations/finance/airbase/vendor-portal-guide
/operations/finance/airbase/frequently-asked-questions
/operations/finance/onboarding
/operations/finance/onboarding/how-to-on-board-as-a-vendor
/operations/finance/onboarding/row-onboarding
/operations/finance/staff-member-expenses
/operations/finance/staff-member-expenses/how-to-spend-company-money
/operations/finance/staff-member-expenses/internship-expense-policy
/operations/finance/staff-member-expenses/corporate-credit-card-policy
/operations/finance/airbase/how-to-access-airbase
/operations/finance/staff-member-expenses/gifting-policy
/operations/finance/staff-member-expenses/airfare-and-traveling
/operations/finance/staff-member-expenses/how-to-reimburse-the-company
/operations/finance/staff-member-expenses/currency-exchange-table
/operations/finance/staff-member-expenses/how-to-get-paid
/operations/finance/how-to-arrange-a-bounty-program
/operations/finance/naming-files-and-agreements
/operations/finance/risk-management
/operations/finance/risk-management/mattermost-templated-agreements
/operations/finance/operations-playbook
/operations/security
/operations/security/policies
/operations/security/privacy
/operations/security/privacy/data-deletion-requests
/operations/security/privacy/data-subject-access-requests
/operations/security/product-security
/operations/security/product-security/product-vulnerability-process
/operations/security/product-security/working-on-sensitive-prs
/operations/security/product-security/secure-software-development
/operations/security/security-operations
/operations/security/security-operations/user-guides
/operations/workplace
/operations/workplace/people
/operations/workplace/people/hr-cadences
/operations/workplace/people/hr-systems
/operations/workplace/people/working-at-mattermost
/operations/workplace/people/working-at-mattermost/onboarding
/operations/workplace/people/working-at-mattermost/onboarding/things-everyone-must-know
/operations/workplace/people/working-at-mattermost/onboarding/staff-onboarding-guide
/operations/workplace/people/working-at-mattermost/onboarding/engineer-onboarding
/operations/workplace/people/working-at-mattermost/onboarding/manager-onboarding-1
/operations/workplace/people/working-at-mattermost/onboarding/frequently-asked-questions
/operations/workplace/people/working-at-mattermost/learning-and-development
/operations/workplace/people/working-at-mattermost/mattermost-communication-best-practices
/operations/workplace/people/working-at-mattermost/paid-time-off
/operations/workplace/people/working-at-mattermost/paid-time-off/ooo-example
/operations/workplace/people/working-at-mattermost/paid-time-off/family-and-friends-day
/operations/workplace/people/working-at-mattermost/travel
/operations/workplace/people/working-at-mattermost/travel/business-travel-insurance
/operations/workplace/people/working-at-mattermost/leaves-of-absence
/operations/workplace/people/working-at-mattermost/leaves-of-absence/pregnancy-leave
/operations/workplace/people/working-at-mattermost/leaves-of-absence/baby-bonding-parental-leave
/operations/workplace/people/working-at-mattermost/leaves-of-absence/jury-duty
/operations/workplace/people/working-at-mattermost/workplace-program
/operations/workplace/people/working-at-mattermost/relocations
/operations/workplace/people/working-at-mattermost/total-rewards
/operations/workplace/people/performance-reviews-50
/operations/workplace/people/performance-reviews-50/formal-review-process
/operations/workplace/people/performance-reviews-50/new-staff-performance-review
/operations/workplace/people/performance-reviews-50/informal-review-process
/operations/workplace/people/internal-transfers
/operations/workplace/people/offboarding
/operations/workplace/people/people-compliance
/operations/workplace/people-policies
/operations/workplace/groups
/operations/workplace/groups/staff-resource-groups
/operations/workplace/approvals-and-iteration
/operations/workplace/it
/operations/workplace/it/helpdesk
/operations/workplace/it/hardware-buy-back-policy
/operations/workplace/it/software-systems
/contributors/contributors
/contributors/contributors/community-edi
/contributors/contributors/ways-to-contribute
/contributors/contributors/ways-to-contribute/community-content-program
/contributors/contributors/ways-to-contribute/documentation-contributions
/contributors/contributors/ways-to-contribute/help-wanted
/contributors/contributors/ways-to-contribute/localization
/contributors/contributors/ways-to-contribute/contribution-events
/contributors/contributors/community
/contributors/contributors/contributor-kindness
/contributors/contributors/community-systems
/contributors/contributors/guidelines
/contributors/contributors/guidelines/social-engagement-guidelines
/contributors/contributors/guidelines/contribution-guidelines
/contributors/contributors/guidelines/community-playbook
/contributors/contributors/guidelines/how-to-run-a-hackathon
/contributors/contributors/guidelines/hacktoberfest-mattermost-organizer-guide
/contributors/mattercon
/contributors/mattercon/people-policies
/contributors/mattercon/mattermost-code-of-conduct
/contributors/mattercon/mattercon2021
/contributors/join-us
/contributors/join-us/ice-breakers
/contributors/join-us/help-wanted
/contributors/contributors/ways-to-contribute/localization
/contributors/join-us/github-sponsorship
/contributors/join-us/recruiting
/contributors/join-us/staff-recruiting
/contributors/join-us/recruiting-cadences
/contributors/join-us/recruiting-cadences/product-manager-hiring
/contributors/join-us/exec-recruiting
/contributors/join-us/exec-recruiting/ea-logistics
/help-and-support/contact-us
https://www.gitbook.com/?utm_source=content&utm_medium=trademark&utm_campaign=-LorODNCmgMboJfZ7MPT-3109903374
https://github.com/mattermost/mattermost-handbook
/company/how-to-guides-for-staff/how-to-update-handbook
https://creativecommons.org/licenses/by-nc-sa/3.0/
#growing-companies-are-a-work-in-progress
/company/about-mattermost/mindsets#drafts-at-1-50-99
/company/about-mattermost/mindsets#drafts-at-1-50-99
#sharing-feedback-and-updating-the-handbook
https://github.com/mattermost/mattermost-handbook/issues
https://community.mattermost.com/private-core/channels/handbook
https://handbook.mattermost.com/company/how-to-guides-for-staff/how-to-update-handbook
#key-links-and-materials
https://mattermost.com/blog/mattermost-v9-0-is-now-available/
https://mattermost.com/newsroom/
https://docs.google.com/document/d/1XGeyPRwD1vr3WYx3bKUGtE1LuW2HgyaZf6Kpf-HjH68/edit
https://docs.google.com/document/d/1BCFdXHMY5iWc6Ix66yuLtLNbO0sYLMJJDDsSLwqvlTA/edit
/company/about-mattermost
#growing-companies-are-a-work-in-progress
#sharing-feedback-and-updating-the-handbook
#key-links-and-materials
https://github.com/mattermost/mattermost-handbook/blob/0.2.1/README.md
/~gitbook/pdf?page=-LorOGxg7ZMopNQHlNZI&only=yes
In [10]:
images_list = []

images = soup.select('img')
for image in images:
	src = image.get('src')
	alt = image.get('alt')
	images_list.append({"src": src, "alt": alt})
	
for image in images_list:
	print(image)

{'src': 'https://handbook.mattermost.com/~gitbook/image?url=https:%2F%2Fhandbook.mattermost.com%2F%7Egitbook%2Ficon%3Fsize=medium%26theme=light&width=32&dpr=4&quality=100&sign=24a6cd98626ece0046782f71d977b7d1821de242ed5f2dc721697e4443a81097', 'alt': 'Logo'}
{'src': 'https://handbook.mattermost.com/~gitbook/image?url=https:%2F%2Fhandbook.mattermost.com%2F%7Egitbook%2Ficon%3Fsize=medium%26theme=dark&width=32&dpr=4&quality=100&sign=12e128ea2b0fc06590fce4ce07b56dfdfe1a4a38a23e1232931fb50101592bd2', 'alt': 'Logo'}
In [24]:
import requests
import csv


website= requests.get('https://handbook.mattermost.com/).text

from bs4 import BeautifulSoup
soup= BeautifulSoup(website, 'lxml')

table= soup.find('table')
table_rows = table.findAll('tr')

data = []
for tr in table_rows:
    td= tr.findAll('td')
    rows = [i.text for i in td]
    data.append(rows)

# Export the data to a CSV file
with open('twitter_accounts.csv', 'w', newline='') as csvfile:
    writer = csv.writer(csvfile)
    writer.writerows(data)

print("Data has been saved to CSV file successfully!")

Data has been saved to CSV file successfully!
In [ ]: