Advanced Web Scraping
& Data Extraction

import requests
from bs4 import BeautifulSoup
import json
import time

# Web scraping with NyronProxies
class WebScraper:
    def __init__(self):
        self.proxies = {
            'http': 'http://username:[email protected]:8000',
            'https': 'http://username:[email protected]:8000'
        }
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
    
    def scrape_page(self, url, parser_func=None):
        try:
            response = requests.get(
                url, 
                proxies=self.proxies, 
                headers=self.headers,
                timeout=30
            )
            response.raise_for_status()
            
            if parser_func:
                return parser_func(response.text)
            else:
                return self.default_parser(response.text)
                
        except requests.exceptions.RequestException as e:
            print(f"Error scraping {url}: {e}")
            return None
    
    def default_parser(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        return {
            'title': soup.find('title').text if soup.find('title') else '',
            'links': [a.get('href') for a in soup.find_all('a', href=True)],
            'text': soup.get_text(strip=True)
        }
    
    def scrape_multiple(self, urls, delay=1):
        results = []
        for url in urls:
            data = self.scrape_page(url)
            if data:
                results.append({'url': url, 'data': data})
            time.sleep(delay)  # Rate limiting
        return results

# Usage
scraper = WebScraper()
urls = ['https://example.com', 'https://example2.com']
results = scraper.scrape_multiple(urls)

for result in results:
    print(f"Scraped {result['url']}: {len(result['data']['links'])} links found")