Web Scraping Best Practices
Introduction to web scraping...
Why Web Scraping?
Web scraping allows you to extract data...
` }, 'ecommerce': { name: 'E-commerce Product', desc: 'Product listing page', html: `
`
},
'blog': {
name: 'Blog Post',
desc: 'Article with metadata',
html: ` Introduction to web scraping... Web scraping allows you to extract data...Web Scraping Best Practices
Why Web Scraping?
| Name | Role | |
|---|---|---|
| Alice Smith | alice@example.com | Admin |
| Bob Johnson | bob@example.com | User |
| Carol White | carol@example.com | Moderator |
` }, 'nested': { name: 'Nested Structure', desc: 'Deeply nested elements', html: `
` } }; function initializeEditor() { const initialContent = `` || templates.simple.html; htmlEditor = CodeMirror(document.getElementById('html_editor'), { value: initialContent, mode: 'xml', lineNumbers: true, theme: 'nord' }); htmlEditor.on('change', debounce(evaluateSelector, 300)); } function loadTemplate(templateKey) { const template = templates[templateKey]; if (template && htmlEditor) { htmlEditor.setValue(template.html); evaluateSelector(); } } function renderTemplates() { const grid = document.getElementById('template_grid'); grid.innerHTML = ''; Object.keys(templates).forEach(key => { const template = templates[key]; const card = document.createElement('div'); card.className = 'template-card'; card.onclick = () => loadTemplate(key); card.innerHTML = `
`; grid.appendChild(card); }); } function switchSelectorType(type) { currentSelectorType = type; document.getElementById('css_btn').classList.toggle('active', type === 'css'); document.getElementById('xpath_btn').classList.toggle('active', type === 'xpath'); evaluateSelector(); } function evalCssSelector(html, selector) { try { const parser = new DOMParser(); const doc = parser.parseFromString(html, 'text/html'); const nodes = doc.querySelectorAll(selector); return Array.from(nodes); } catch (e) { throw new Error(`CSS Selector Error: ${e.message}`); } } function evalXPathSelector(html, selector) { try { const parser = new DOMParser(); const doc = parser.parseFromString(html, 'text/html'); const result = doc.evaluate(selector, doc, null, XPathResult.ANY_TYPE, null); const nodes = []; let node; // Handle different XPath result types if ([4, 5, 6, 7, 8, 0].includes(result.resultType)) { while ((node = result.iterateNext())) { nodes.push(node); } } else if (result.resultType === 1) { // NUMBER_TYPE return [{ type: 'number', value: result.numberValue }]; } else if (result.resultType === 2) { // STRING_TYPE return [{ type: 'string', value: result.stringValue }]; } else if (result.resultType === 3) { // BOOLEAN_TYPE return [{ type: 'boolean', value: result.booleanValue }]; } else if (result.singleNodeValue) { nodes.push(result.singleNodeValue); } return nodes; } catch (e) { throw new Error(`XPath Error: ${e.message}`); } } function nodeToString(node) { if (node.type) { // Primitive type (number, string, boolean) return String(node.value); } if (node instanceof HTMLElement) { return node.outerHTML; } else if (node.nodeType === 2) { // Attribute return node.value; } else if (node.nodeType === 3) { // Text return node.wholeText; } else if (node.nodeType === 8) { // Comment return ``; } return String(node); } function evaluateSelector() { currentSelector = document.getElementById('selector_input').value.trim(); if (!currentSelector || !htmlEditor) { matchedResults = []; renderResults(); return; } const html = htmlEditor.getValue(); try { if (currentSelectorType === 'css') { matchedResults = evalCssSelector(html, currentSelector); } else { matchedResults = evalXPathSelector(html, currentSelector); } renderResults(); } catch (e) { matchedResults = []; renderResults(e.message); } } function renderResults(error = null) { const resultsList = document.getElementById('results_list'); const matchCount = document.getElementById('match_count'); if (error) { resultsList.innerHTML = `
`; matchCount.textContent = '0'; return; } matchCount.textContent = matchedResults.length; if (matchedResults.length === 0) { resultsList.innerHTML = '
'; return; } resultsList.innerHTML = ''; matchedResults.forEach((node, index) => { const item = document.createElement('div'); item.className = 'result-item'; const content = nodeToString(node); // Create header with index, node name, and copy button const headerDiv = document.createElement('div'); headerDiv.className = 'result-item-header'; const indexSpan = document.createElement('span'); indexSpan.className = 'result-index'; indexSpan.textContent = index + 1; const nodeNameSpan = document.createElement('span'); nodeNameSpan.className = 'result-node-name'; nodeNameSpan.textContent = node.nodeName || 'Value'; const copyBtn = document.createElement('button'); copyBtn.className = 'result-copy-btn'; copyBtn.innerHTML = ''; copyBtn.title = 'Copy this result'; copyBtn.onclick = (e) => { e.stopPropagation(); navigator.clipboard.writeText(content).then(() => { copyBtn.innerHTML = ''; setTimeout(() => { copyBtn.innerHTML = ''; }, 1500); }); }; headerDiv.appendChild(indexSpan); headerDiv.appendChild(document.createTextNode(' ')); headerDiv.appendChild(nodeNameSpan); headerDiv.appendChild(copyBtn); const contentDiv = document.createElement('div'); contentDiv.className = 'result-content'; contentDiv.textContent = content.substring(0, 200) + (content.length > 200 ? '...' : ''); item.appendChild(headerDiv); item.appendChild(contentDiv); resultsList.appendChild(item); }); } function copyAllResults() { const results = matchedResults.map(nodeToString).join('\n\n'); navigator.clipboard.writeText(results).then(() => { alert('Results copied to clipboard!'); }); } function exportPython() { const codeDiv = document.getElementById('code_export'); let code = ''; if (currentSelectorType === 'css') { code = `from lxml import html # Parse HTML tree = html.fromstring(html_content) # CSS Selector (using cssselect) elements = tree.cssselect('${currentSelector.replace(/'/g, "\\'")}') # Extract text or attributes results = [elem.text_content() for elem in elements] # Or get HTML results = [html.tostring(elem, encoding='unicode') for elem in elements]`; } else { code = `from lxml import html # Parse HTML tree = html.fromstring(html_content) # XPath Selector elements = tree.xpath('${currentSelector.replace(/'/g, "\\'")}') # Extract results results = [elem if isinstance(elem, str) else elem.text_content() for elem in elements] # Or get HTML (for elements) results = [html.tostring(elem, encoding='unicode') if hasattr(elem, 'tag') else str(elem) for elem in elements]`; } showCodeExport(code, 'Python'); } function exportJavaScript() { let code = ''; if (currentSelectorType === 'css') { code = `// Using Puppeteer const elements = await page.$$('${currentSelector.replace(/'/g, "\\'")}'); const results = await Promise.all( elements.map(el => el.evaluate(node => node.textContent)) ); // Using Cheerio const $ = cheerio.load(html); const results = $('${currentSelector.replace(/'/g, "\\'")}') .map((i, el) => $(el).text()) .get(); // Vanilla JavaScript (browser) const elements = document.querySelectorAll('${currentSelector.replace(/'/g, "\\'")}'); const results = Array.from(elements).map(el => el.textContent);`; } else { code = `// Using Puppeteer with XPath const elements = await page.$x('${currentSelector.replace(/'/g, "\\'")}'); const results = await Promise.all( elements.map(el => el.evaluate(node => node.nodeType === 1 ? node.textContent : node.nodeValue )) ); // Vanilla JavaScript (browser) const result = document.evaluate( '${currentSelector.replace(/'/g, "\\'")}', document, null, XPathResult.ANY_TYPE, null ); const results = []; let node; while (node = result.iterateNext()) { results.push(node.textContent || node.nodeValue); }`; } showCodeExport(code, 'JavaScript'); } function showCodeExport(code, language) { const codeDiv = document.getElementById('code_export'); codeDiv.innerHTML = `
${escapeHtml(code)}`; codeDiv.classList.remove('d-none'); // Add click handler for copy button document.getElementById('copy-code-btn').addEventListener('click', function() { navigator.clipboard.writeText(code).then(() => { this.innerHTML = ' Copied!'; setTimeout(() => { this.innerHTML = ' Copy'; }, 2000); }); }); } function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; } function downloadResults() { const results = matchedResults.map(nodeToString).join('\n\n'); const blob = new Blob([results], { type: 'text/plain' }); const url = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = url; a.download = 'selector_results.txt'; a.click(); URL.revokeObjectURL(url); } function clearHTML() { if (confirm('Clear HTML editor?')) { htmlEditor.setValue(''); evaluateSelector(); } } function debounce(func, wait) { let timeout; return function(...args) { clearTimeout(timeout); timeout = setTimeout(() => func.apply(this, args), wait); }; } // Initialize on page load document.addEventListener('DOMContentLoaded', () => { renderTemplates(); initializeEditor(); // Auto-detect selector type const selectorInput = document.getElementById('selector_input'); if (selectorInput.value && selectorInput.value.startsWith('/')) { switchSelectorType('xpath'); } else if (selectorInput.value) { switchSelectorType('css'); } evaluateSelector(); // Event listeners selectorInput.addEventListener('input', debounce(evaluateSelector, 300)); document.getElementById('css_btn').addEventListener('click', () => switchSelectorType('css')); document.getElementById('xpath_btn').addEventListener('click', () => switchSelectorType('xpath')); });
