feat: add rendering to the sdk

VinciGit00 · VinciGit00 · commit 3c9770a76f3d · 2025-11-26T08:04:44.000-08:00
diff --git a/scrapegraph-js/examples/utilities/scrape_example.js b/scrapegraph-js/examples/utilities/scrape_example.js
@@ -33,17 +33,19 @@ const OUTPUT_DIR = 'scrape_output';
  * @returns {Object} The API response containing HTML content and metadata
  */
 async function scrapeWebsite(websiteUrl, options = {}) {
-  const { renderHeavyJs = false, headers = {} } = options;
+  const { renderHeavyJs = false, branding = false, headers = {} } = options;
   
   const jsMode = renderHeavyJs ? 'with heavy JS rendering' : 'without JS rendering';
+  const brandingMode = branding ? 'with branding' : 'without branding';
   console.log(`Getting HTML content from: ${websiteUrl}`);
-  console.log(`Mode: ${jsMode}`);
+  console.log(`Mode: ${jsMode}, ${brandingMode}`);
 
   const startTime = Date.now();
   
   try {
     const result = await scrape(API_KEY, websiteUrl, {
       renderHeavyJs,
+      branding,
       headers
     });
     
@@ -118,12 +120,21 @@ async function main() {
       url: 'https://example.com',
       name: 'example',
       renderHeavyJs: false,
+      branding: false,
       description: 'Simple static website',
     },
+    {
+      url: 'https://www.cubic.dev/',
+      name: 'cubic_dev',
+      renderHeavyJs: false,
+      branding: true,
+      description: 'Website with branding enabled',
+    },
     {
       url: 'https://httpbin.org/html',
       name: 'httpbin_html',
       renderHeavyJs: false,
+      branding: false,
       description: 'HTTP testing service',
     },
   ];
@@ -147,7 +158,8 @@ async function main() {
     try {
       // Get HTML content
       const result = await scrapeWebsite(website.url, {
-        renderHeavyJs: website.renderHeavyJs
+        renderHeavyJs: website.renderHeavyJs,
+        branding: website.branding
       });
 
       // Display response metadata
@@ -174,7 +186,8 @@ async function main() {
         console.log(`  Link tags: ${stats.linkTags}`);
 
         // Save HTML content
-        const filename = `${website.name}_${website.renderHeavyJs ? 'js' : 'nojs'}`;
+        const brandingSuffix = website.branding ? '_branding' : '';
+        const filename = `${website.name}_${website.renderHeavyJs ? 'js' : 'nojs'}${brandingSuffix}`;
         await saveHtmlContent(htmlContent, filename);
 
         // Show first 500 characters as preview
diff --git a/scrapegraph-js/package.json b/scrapegraph-js/package.json
@@ -1,7 +1,7 @@
 {
   "name": "scrapegraph-js",
   "author": "ScrapeGraphAI",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "Scrape and extract structured data from a webpage using ScrapeGraphAI's APIs. Supports cookies for authentication, infinite scrolling, and pagination.",
   "repository": {
     "type": "git",
diff --git a/scrapegraph-js/src/scrape.js b/scrapegraph-js/src/scrape.js
@@ -10,6 +10,7 @@ import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.j
  * @param {string} url - The URL of the webpage to be converted.
  * @param {Object} options - Optional configuration options.
  * @param {boolean} options.renderHeavyJs - Whether to render heavy JavaScript (defaults to false).
+ * @param {boolean} [options.branding=false] - Whether to include branding in the response (defaults to false).
  * @param {Object} options.headers - Optional custom headers to send with the request.
  * @param {boolean} [options.stealth=false] - Enable stealth mode to avoid bot detection
  * @returns {Promise<Object>} A promise that resolves to the HTML content and metadata.
@@ -35,6 +36,12 @@ import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.j
  * });
  *
  * @example
+ * // With branding enabled:
+ * const result = await scrape(apiKey, url, {
+ *   branding: true
+ * });
+ *
+ * @example
  * // With custom headers:
  * const result = await scrape(apiKey, url, {
  *   renderHeavyJs: false,
@@ -47,6 +54,7 @@ import { getMockResponse, createMockAxiosResponse } from './utils/mockResponse.j
 export async function scrape(apiKey, url, options = {}) {
   const {
     renderHeavyJs = false,
+    branding = false,
     headers: customHeaders = {},
     mock = null,
     stealth = false
@@ -75,6 +83,10 @@ export async function scrape(apiKey, url, options = {}) {
     render_heavy_js: renderHeavyJs,
   };
 
+  if (branding) {
+    payload.branding = branding;
+  }
+
   if (stealth) {
     payload.stealth = stealth;
   }
diff --git a/scrapegraph-py/examples/scrape/async/async_scrape_example.py b/scrapegraph-py/examples/scrape/async/async_scrape_example.py
@@ -5,15 +5,17 @@
 1. How to make async scrape requests
 2. How to process multiple URLs concurrently
 3. How to use render_heavy_js for JavaScript-heavy websites
-4. How to add custom headers in async mode
+4. How to use branding parameter
+5. How to add custom headers in async mode
 
 Equivalent curl command:
 curl -X POST https://api.scrapegraphai.com/v1/scrape \
   -H "Content-Type: application/json" \
   -H "SGAI-APIKEY: your-api-key-here" \
   -d '{
-    "website_url": "https://example.com",
-    "render_heavy_js": false
+    "website_url": "https://www.cubic.dev/",
+    "render_heavy_js": false,
+    "branding": true
   }'
 
 Requirements:
@@ -165,6 +167,31 @@ async def concurrent_scraping_example():
         return results
 
 
+async def async_scrape_with_branding():
+    """Demonstrate async scraping with branding enabled."""
+    print("\n🏷️ Async Branding Example")
+    print("=" * 30)
+    
+    async with AsyncClient.from_env() as client:
+        try:
+            print("Making async scrape request with branding enabled...")
+            result = await client.scrape(
+                website_url="https://www.cubic.dev/",
+                render_heavy_js=False,
+                branding=True
+            )
+            
+            html_content = result.get("html", "")
+            print(f"✅ Success! Received {len(html_content):,} characters of HTML")
+            print(f"Request ID: {result.get('request_id', 'N/A')}")
+            
+            return result
+            
+        except Exception as e:
+            print(f"❌ Error: {str(e)}")
+            return None
+
+
 async def async_scrape_with_custom_headers():
     """Demonstrate async scraping with custom headers."""
     print("\n🔧 Async Custom Headers Example")
@@ -228,7 +255,17 @@ def demonstrate_curl_equivalent():
     print("    \"render_heavy_js\": false")
     print("  }'")
     
-    print("\n2. Multiple concurrent requests:")
+    print("\n2. With branding enabled:")
+    print("curl -X POST https://api.scrapegraphai.com/v1/scrape \\")
+    print("  -H \"Content-Type: application/json\" \\")
+    print("  -H \"SGAI-APIKEY: your-api-key-here\" \\")
+    print("  -d '{")
+    print("    \"website_url\": \"https://www.cubic.dev/\",")
+    print("    \"render_heavy_js\": false,")
+    print("    \"branding\": true")
+    print("  }'")
+    
+    print("\n3. Multiple concurrent requests:")
     print("# Run multiple curl commands in parallel:")
     print("curl -X POST https://api.scrapegraphai.com/v1/scrape \\")
     print("  -H \"Content-Type: application/json\" \\")
@@ -253,7 +290,8 @@ async def main():
         # Run async examples
         result1 = await basic_async_scrape()
         result2 = await async_scrape_with_heavy_js()
-        result3 = await async_scrape_with_custom_headers()
+        result3 = await async_scrape_with_branding()
+        result4 = await async_scrape_with_custom_headers()
         concurrent_results = await concurrent_scraping_example()
         
         # Save results if successful
@@ -265,12 +303,18 @@ async def main():
         if result3:
             html3 = result3.get("html", "")
             if html3:
-                await save_html_to_file_async(html3, "custom_headers_async_scrape")
+                await save_html_to_file_async(html3, "branding_async_scrape")
+        
+        if result4:
+            html4 = result4.get("html", "")
+            if html4:
+                await save_html_to_file_async(html4, "custom_headers_async_scrape")
         
         print("\n🎯 Summary:")
         print(f"✅ Basic async scrape: {'Success' if result1 else 'Failed'}")
         print(f"✅ Heavy JS async scrape: {'Success' if result2 else 'Failed'}")
-        print(f"✅ Custom headers async scrape: {'Success' if result3 else 'Failed'}")
+        print(f"✅ Branding async scrape: {'Success' if result3 else 'Failed'}")
+        print(f"✅ Custom headers async scrape: {'Success' if result4 else 'Failed'}")
         print(f"✅ Concurrent scraping: {'Success' if concurrent_results else 'Failed'}")
         
     except Exception as e:
diff --git a/scrapegraph-py/examples/scrape/sync/scrape_example.py b/scrapegraph-py/examples/scrape/sync/scrape_example.py
@@ -4,16 +4,18 @@
 This example shows:
 1. How to make a basic scrape request
 2. How to use render_heavy_js for JavaScript-heavy websites
-3. How to add custom headers
-4. How to handle the response
+3. How to use branding parameter
+4. How to add custom headers
+5. How to handle the response
 
 Equivalent curl command:
 curl -X POST https://api.scrapegraphai.com/v1/scrape \
   -H "Content-Type: application/json" \
   -H "SGAI-APIKEY: your-api-key-here" \
   -d '{
     "website_url": "https://example.com",
-    "render_heavy_js": false
+    "render_heavy_js": false,
+    "branding": true
   }'
 
 Requirements:
@@ -98,6 +100,38 @@ def scrape_with_heavy_js():
         client.close()
 
 
+def scrape_with_branding():
+    """Demonstrate scraping with branding enabled."""
+    print("\n🏷️ Branding Example")
+    print("=" * 30)
+    
+    client = Client.from_env()
+    
+    try:
+        print("Making scrape request with branding enabled...")
+        result = client.scrape(
+            website_url="https://www.cubic.dev/",
+            render_heavy_js=False,
+            branding=True
+        )
+        
+        html_content = result.get("html", "")
+        print(f"✅ Success! Received {len(html_content):,} characters of HTML")
+        print(f"Request ID: {result.get('request_id', 'N/A')}")
+        
+        # Show a preview of the HTML
+        preview = html_content[:200].replace('\n', ' ').strip()
+        print(f"HTML Preview: {preview}...")
+        
+        return result
+        
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        return None
+    finally:
+        client.close()
+
+
 def scrape_with_custom_headers():
     """Demonstrate scraping with custom headers."""
     print("\n🔧 Custom Headers Example")
@@ -175,6 +209,16 @@ def demonstrate_curl_equivalent():
     print("    \"website_url\": \"https://example.com\",")
     print("    \"render_heavy_js\": true")
     print("  }'")
+    
+    print("\n3. With branding enabled:")
+    print("curl -X POST https://api.scrapegraphai.com/v1/scrape \\")
+    print("  -H \"Content-Type: application/json\" \\")
+    print("  -H \"SGAI-APIKEY: your-api-key-here\" \\")
+    print("  -d '{")
+    print("    \"website_url\": \"https://www.cubic.dev/\",")
+    print("    \"render_heavy_js\": false,")
+    print("    \"branding\": true")
+    print("  }'")
 
 
 def main():
@@ -189,7 +233,8 @@ def main():
         # Run examples
         result1 = basic_scrape_example()
         result2 = scrape_with_heavy_js()
-        result3 = scrape_with_custom_headers()
+        result3 = scrape_with_branding()
+        result4 = scrape_with_custom_headers()
         
         # Save results if successful
         if result1:
@@ -200,12 +245,18 @@ def main():
         if result3:
             html3 = result3.get("html", "")
             if html3:
-                save_html_to_file(html3, "custom_headers_scrape")
+                save_html_to_file(html3, "branding_scrape")
+        
+        if result4:
+            html4 = result4.get("html", "")
+            if html4:
+                save_html_to_file(html4, "custom_headers_scrape")
         
         print("\n🎯 Summary:")
         print(f"✅ Basic scrape: {'Success' if result1 else 'Failed'}")
         print(f"✅ Heavy JS scrape: {'Success' if result2 else 'Failed'}")
-        print(f"✅ Custom headers scrape: {'Success' if result3 else 'Failed'}")
+        print(f"✅ Branding scrape: {'Success' if result3 else 'Failed'}")
+        print(f"✅ Custom headers scrape: {'Success' if result4 else 'Failed'}")
         
     except Exception as e:
         print(f"❌ Unexpected error: {str(e)}")
diff --git a/scrapegraph-py/scrapegraph_py/async_client.py b/scrapegraph-py/scrapegraph_py/async_client.py
@@ -481,6 +481,7 @@ async def scrape(
         self,
         website_url: str,
         render_heavy_js: bool = False,
+        branding: bool = False,
         headers: Optional[dict[str, str]] = None,
         stealth: bool = False,
     ):
@@ -489,11 +490,13 @@ async def scrape(
         Args:
             website_url: The URL of the website to get HTML from
             render_heavy_js: Whether to render heavy JavaScript (defaults to False)
+            branding: Whether to include branding in the response (defaults to False)
             headers: Optional headers to send with the request
             stealth: Enable stealth mode to avoid bot detection
         """
         logger.info(f"🔍 Starting scrape request for {website_url}")
         logger.debug(f"🔧 Render heavy JS: {render_heavy_js}")
+        logger.debug(f"🔧 Branding: {branding}")
         if headers:
             logger.debug("🔧 Using custom headers")
         if stealth:
@@ -502,6 +505,7 @@ async def scrape(
         request = ScrapeRequest(
             website_url=website_url,
             render_heavy_js=render_heavy_js,
+            branding=branding,
             headers=headers,
             stealth=stealth,
         )
diff --git a/scrapegraph-py/scrapegraph_py/client.py b/scrapegraph-py/scrapegraph_py/client.py
@@ -491,6 +491,7 @@ def scrape(
         self,
         website_url: str,
         render_heavy_js: bool = False,
+        branding: bool = False,
         headers: Optional[dict[str, str]] = None,
         mock:bool=False,
         stealth:bool=False,
@@ -500,11 +501,13 @@ def scrape(
         Args:
             website_url: The URL of the website to get HTML from
             render_heavy_js: Whether to render heavy JavaScript (defaults to False)
+            branding: Whether to include branding in the response (defaults to False)
             headers: Optional headers to send with the request
             stealth: Enable stealth mode to avoid bot detection
         """
         logger.info(f"🔍 Starting scrape request for {website_url}")
         logger.debug(f"🔧 Render heavy JS: {render_heavy_js}")
+        logger.debug(f"🔧 Branding: {branding}")
         if headers:
             logger.debug("🔧 Using custom headers")
         if stealth:
@@ -513,6 +516,7 @@ def scrape(
         request = ScrapeRequest(
             website_url=website_url,
             render_heavy_js=render_heavy_js,
+            branding=branding,
             headers=headers,
             mock=mock,
             stealth=stealth
diff --git a/scrapegraph-py/scrapegraph_py/models/scrape.py b/scrapegraph-py/scrapegraph_py/models/scrape.py

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "scrapegraph-js",`
`3`	`3`	`"author": "ScrapeGraphAI",`
`4`		`- "version": "0.2.4",`
	`4`	`+ "version": "0.2.5",`
`5`	`5`	`"description": "Scrape and extract structured data from a webpage using ScrapeGraphAI's APIs. Supports cookies for authentication, infinite scrolling, and pagination.",`
`6`	`6`	`"repository": {`
`7`	`7`	`"type": "git",`