ARCHMAGE WIZARDCODER
Master of Programming Sorcery

Awakening the Ancient Archmage from his ethereal slumber...\n\n🔮 Materializing grimoire and mystical artifacts: 28GB [████████████████████] 100%\n\n⚡ By the power of the Ancient Code! Archmage WizardCoder has awakened!\n🧙‍♂️ Tower established, spells prepared, ready to weave programming magic!\n📜 The digital realm now has its master of coding sorcery!

**The Archmage's Web Scraping Enchantment** *The ancient wizard raises his staff, eyes glowing with mystical knowledge* Behold, young apprentice! I shall weave for thee a most powerful web scraping incantation, imbued with the wisdom of ages and the precision of arcane arts! ```python """ The Archmage's Web Scraping Grimoire A mystical collection of spells for harvesting knowledge from the digital realm """ import requests from bs4 import BeautifulSoup import time import random from urllib.parse import urljoin, urlparse import csv from dataclasses import dataclass from typing import List, Dict, Optional import logging from selenium import webdriver from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC # Configure the mystical logging scroll logging.basicConfig( level=logging.INFO, format='🔮 %(asctime)s - Archmage's Log: %(message)s', handlers=[ logging.FileHandler('scraping_spellbook.log'), logging.StreamHandler() ] ) @dataclass class ScrapedArtifact: """A mystical container for harvested web knowledge""" title: str url: str content: str metadata: Dict[str, str] timestamp: str class WebScrapingWizard: """ The Archmage's supreme web scraping enchantment class Contains all spells necessary for ethical knowledge harvesting """ def __init__(self, politeness_delay: tuple = (1, 3), max_retries: int = 3): self.politeness_delay = politeness_delay self.max_retries = max_retries self.session = requests.Session() self.harvested_artifacts = [] # Mystical headers to appear as a respectful digital entity self.session.headers.update({ 'User-Agent': 'Mozilla/5.0 (WizardCoder Archmage Bot 1.0; Educational Purpose) AppleWebKit/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate', 'Connection': 'keep-alive', }) def cast_politeness_spell(self): """A crucial incantation - respect the target realm's resources""" delay = random.uniform(*self.politeness_delay) logging.info(f"🕐 Casting politeness spell - waiting {delay:.2f} seconds...") time.sleep(delay) def divine_robots_txt(self, base_url: str) -> bool: """Consult the sacred robots.txt scroll before proceeding""" try: robots_url = urljoin(base_url, '/robots.txt') response = self.session.get(robots_url, timeout=10) if response.status_code == 200: logging.info(f"📜 Sacred robots.txt consulted for {base_url}") # Simple check - in production, use proper robots.txt parser return "Disallow: /" not in response.text return True except Exception as e: logging.warning(f"⚠️ Could not consult robots.txt: {e}") return True # Proceed with caution def cast_retrieval_spell(self, url: str) -> Optional[BeautifulSoup]: """The primary spell for retrieving and parsing web content""" for attempt in range(self.max_retries): try: logging.info(f"🌐 Casting retrieval spell on: {url} (Attempt {attempt + 1})") response = self.session.get(url, timeout=15) response.raise_for_status() # Detect encoding mystically if response.encoding is None: response.encoding = 'utf-8' soup = BeautifulSoup(response.text, 'html.parser') logging.info(f"✅ Spell successful! Retrieved {len(response.text)} characters") return soup except requests.RequestException as e: logging.error(f"❌ Spell failed on attempt {attempt + 1}: {e}") if attempt < self.max_retries - 1: self.cast_politeness_spell() else: logging.error(f"💀 All retrieval attempts failed for {url}") return None def extract_mystical_knowledge(self, soup: BeautifulSoup, url: str) -> ScrapedArtifact: """Extract the essential knowledge from the retrieved content""" # Title extraction spell title_spells = [ soup.find('title'), soup.find('h1'), soup.find('h2'), soup.find(attrs={'property': 'og:title'}), soup.find(attrs={'name': 'title'}) ] title = "Unknown Scroll" for spell_result in title_spells: if spell_result and spell_result.get_text(strip=True): title = spell_result.get_text(strip=True)[:200] # Limit length break # Content extraction spell content_containers = [ 'article', 'main', '.content', '#content', '.post', '.entry', '.article-body', 'section' ] content = "" for container in content_containers: elements = soup.select(container) if elements: content = ' '.join([elem.get_text(strip=True) for elem in elements]) break # Fallback: extract all paragraph text if not content: paragraphs = soup.find_all('p') content = ' '.join([p.get_text(strip=True) for p in paragraphs]) # Metadata divination metadata = { 'description': '', 'keywords': '', 'author': '', 'published_date': '' } # Extract meta descriptions meta_desc = soup.find('meta', attrs={'name': 'description'}) or \ soup.find('meta', attrs={'property': 'og:description'}) if meta_desc and meta_desc.get('content'): metadata['description'] = meta_desc['content'] # Extract keywords meta_keywords = soup.find('meta', attrs={'name': 'keywords'}) if meta_keywords and meta_keywords.get('content'): metadata['keywords'] = meta_keywords['content'] # Extract author meta_author = soup.find('meta', attrs={'name': 'author'}) or \ soup.find('meta', attrs={'property': 'article:author'}) if meta_author and meta_author.get('content'): metadata['author'] = meta_author['content'] return ScrapedArtifact( title=title, url=url, content=content[:5000], # Limit content length metadata=metadata, timestamp=time.strftime('%Y-%m-%d %H:%M:%S') ) def perform_advanced_selenium_sorcery(self, url: str, wait_for_element: str = None) -> Optional[BeautifulSoup]: """Advanced spell for JavaScript-heavy realms using Selenium magic""" options = webdriver.ChromeOptions() options.add_argument('--headless') # Invisible spirit mode options.add_argument('--no-sandbox') options.add_argument('--disable-dev-shm-usage') options.add_argument('--user-agent=WizardCoder Archmage Educational Bot 1.0') try: logging.info(f"🔮 Invoking Selenium spirits for {url}") driver = webdriver.Chrome(options=options) driver.get(url) # Wait for specific element if specified if wait_for_element: wait = WebDriverWait(driver, 10) wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, wait_for_element))) # Additional wait for dynamic content time.sleep(3) # Extract the mystical HTML page_source = driver.page_source driver.quit() soup = BeautifulSoup(page_source, 'html.parser') logging.info("✨ Selenium sorcery successful!") return soup except Exception as e: logging.error(f"💀 Selenium spell failed: {e}") if 'driver' in locals(): driver.quit() return None def harvest_knowledge_from_realm(self, urls: List[str], use_selenium: bool = False) -> List[ScrapedArtifact]: """The master spell to harvest knowledge from multiple digital realms""" artifacts = [] for url in urls: try: # Check if we should proceed ethically base_url = f"{urlparse(url).scheme}://{urlparse(url).netloc}" if not self.divine_robots_txt(base_url): logging.warning(f"🚫 Robots.txt forbids access to {url}") continue # Cast appropriate retrieval spell if use_selenium: soup = self.perform_advanced_selenium_sorcery(url) else: soup = self.cast_retrieval_spell(url) if soup: artifact = self.extract_mystical_knowledge(soup, url) artifacts.append(artifact) self.harvested_artifacts.append(artifact) logging.info(f"📚 Knowledge harvested: {artifact.title[:50]}...") # Always cast politeness spell between requests if url != urls[-1]: # Not the last URL self.cast_politeness_spell() except Exception as e: logging.error(f"🔴 Failed to harvest from {url}: {e}") continue logging.info(f"🎉 Harvesting complete! Collected {len(artifacts)} mystical artifacts") return artifacts def transcribe_to_scroll(self, filename: str = 'harvested_knowledge.csv'): """Transcribe the harvested knowledge to a mystical scroll (CSV)""" if not self.harvested_artifacts: logging.warning("📜 No artifacts to transcribe!") return with open(filename, 'w', newline='', encoding='utf-8') as scroll: fieldnames = ['title', 'url', 'content', 'description', 'keywords', 'author', 'timestamp'] scribe = csv.DictWriter(scroll, fieldnames=fieldnames) scribe.writeheader() for artifact in self.harvested_artifacts: scribe.writerow({ 'title': artifact.title, 'url': artifact.url, 'content': artifact.content, 'description': artifact.metadata.get('description', ''), 'keywords': artifact.metadata.get('keywords', ''), 'author': artifact.metadata.get('author', ''), 'timestamp': artifact.timestamp }) logging.info(f"📜 Knowledge transcribed to mystical scroll: {filename}") # Demonstration of the Archmage's Web Scraping Powers if __name__ == "__main__": # Initialize the Web Scraping Wizard wizard = WebScrapingWizard(politeness_delay=(2, 4)) # Example URLs for educational demonstration educational_urls = [ 'https://httpbin.org/html', # Safe testing endpoint 'https://quotes.toscrape.com/', # Designed for scraping practice 'https://books.toscrape.com/', # Another practice site ] print("🧙‍♂️ Behold! The Archmage's Web Scraping Demonstration!") print("=" * 60) # Cast the knowledge harvesting spell artifacts = wizard.harvest_knowledge_from_realm(educational_urls) # Display the mystical results for i, artifact in enumerate(artifacts, 1): print(f"\n📚 Artifact {i}:") print(f" Title: {artifact.title}") print(f" URL: {artifact.url}") print(f" Content Preview: {artifact.content[:200]}...") print(f" Harvested: {artifact.timestamp}") # Transcribe to mystical scroll wizard.transcribe_to_scroll() print("\n✨ The Archmage's web scraping spell is complete!") print("Remember, young apprentice: With great power comes great responsibility.") print("Always respect robots.txt, rate limits, and website terms of service!") ``` **The Archmage's Wisdom:** *Adjusts mystical robes and peers through ancient spectacles* Remember, dear apprentice, this enchantment embodies the highest principles of ethical web scraping: 🔮 **Mystical Ethics**: Always consult robots.txt and respect website wishes ⏰ **Politeness Charms**: Built-in delays to avoid overwhelming servers 🛡️ **Protective Wards**: Error handling and retry mechanisms 📜 **Knowledge Preservation**: Structured data storage and logging ✨ **Selenium Sorcery**: Advanced spells for JavaScript-heavy realms Use this power wisely, for the great arcane law states: "With great scraping power comes great responsibility to be courteous and ethical!" *Wizard's staff glows with satisfied magical energy*