[Luke's Contribution] Telegram Group Scraper in Python ;)

undetected-chromedriver==3.5.4
pandas==2.1.4

import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import json
import pandas as pd
import os
import re

class TelegramChannelScraper:
    def __init__(self):
        self.setup_driver()
        self.channels = []
        
    def setup_driver(self):
        try:
            options = uc.ChromeOptions()
            options.add_argument('--no-sandbox')
            options.add_argument('--disable-dev-shm-usage')
            options.add_argument('--window-size=1920,1080')
            options.add_argument('--disable-blink-features=AutomationControlled')
            options.add_argument('--disable-notifications')
            
            # Specify the Chrome version explicitly
            self.driver = uc.Chrome(
                options=options,
                driver_executable_path=None,
                browser_executable_path=None,
                version_main=136  # Set to match your Chrome version
            )
            
            self.driver.set_page_load_timeout(30)
            print("Chrome driver initialized successfully")
        except Exception as e:
            print(f"Error initializing Chrome driver: {str(e)}")
            raise

    def extract_member_count(self, element):
        try:
            # First try to find number in the text content
            text = element.text
            matches = re.findall(r'(\d+(?:,\d+)*(?:\s*k)?)', text.lower())
            if matches:
                # Process each match
                max_number = 0
                for match in matches:
                    try:
                        # Handle 'k' suffix
                        if 'k' in match.lower():
                            num = float(match.lower().replace('k', '').replace(',', '')) * 1000
                        else:
                            num = float(match.replace(',', ''))
                        max_number = max(max_number, num)
                    except:
                        continue
                if max_number > 0:
                    return int(max_number)  # Return as integer
        except:
            pass
        
        return 0  # Return 0 instead of "N/A" for numeric consistency
        
    def scroll_to_end(self):
        print("Scrolling to the end of the page...")
        scroll_pause_time = 0.2
        scroll_attempts = 0
        max_scroll_attempts = 850  # Changed to 600
        last_height = self.driver.execute_script("return document.body.scrollHeight")
        no_new_height_count = 0
        
        while scroll_attempts < max_scroll_attempts and no_new_height_count < 3:
            scroll_attempts += 1
            if scroll_attempts % 50 == 0:
                print(f"Scroll attempt {scroll_attempts}/{max_scroll_attempts}")
            
            # Fast scroll to bottom
            self.driver.execute_script("window.scrollTo({top: document.body.scrollHeight, behavior: 'auto'});")
            time.sleep(scroll_pause_time)
            
            # Calculate new scroll height
            new_height = self.driver.execute_script("return document.body.scrollHeight")
            if new_height == last_height:
                no_new_height_count += 1
            else:
                no_new_height_count = 0
            last_height = new_height
        
        if scroll_attempts >= max_scroll_attempts:
            print("Reached maximum scroll attempts")
        else:
            print("Reached end of page")
        
    def extract_all_channels(self):
        print("Extracting all channels...")
        try:
            # Get all channel elements at once
            channel_elements = self.driver.find_elements(By.CSS_SELECTOR, "div.info.flex-grow-1")
            total_elements = len(channel_elements)
            print(f"Found {total_elements} elements to process")
            
            # Pre-compile regex pattern
            number_pattern = re.compile(r'(\d+(?:,\d+)*(?:\s*k)?)')
            
            # Process in batches for faster execution
            batch_size = 100
            new_channels = []
            existing_links = {channel['link'] for channel in self.channels}  # For faster duplicate checking
            
            for batch_start in range(0, total_elements, batch_size):
                batch_end = min(batch_start + batch_size, total_elements)
                batch = channel_elements[batch_start:batch_end]
                
                for element in batch:
                    try:
                        # Extract all data at once to minimize DOM interactions
                        title_element = element.find_element(By.CSS_SELECTOR, "span.card-title")
                        link_element = element.find_element(By.CSS_SELECTOR, "a")
                        
                        title = title_element.text
                        link = link_element.get_attribute("href")
                        
                        # Skip if we already have this link
                        if link in existing_links:
                            continue
                        
                        # Fast member count extraction
                        text = element.text.lower()
                        matches = number_pattern.findall(text)
                        members = 0
                        if matches:
                            try:
                                max_number = 0
                                for match in matches:
                                    if 'k' in match:
                                        num = float(match.replace('k', '').replace(',', '')) * 1000
                                    else:
                                        num = float(match.replace(',', ''))
                                    max_number = max(max_number, num)
                                members = int(max_number)
                            except:
                                pass
                        
                        channel_data = {
                            "title": title,
                            "link": link,
                            "members": members
                        }
                        
                        new_channels.append(channel_data)
                        existing_links.add(link)
                        
                    except Exception as e:
                        continue
                
                # Update progress and save after each batch
                current_total = len(new_channels)
                print(f"Processed {batch_end}/{total_elements} elements - Found {current_total} new channels")
                
                # Add batch to main list and save if we have enough new channels
                if new_channels:
                    self.channels.extend(new_channels)
                    if len(new_channels) >= 100:
                        self.save_results(is_final=False)
                    new_channels = []
            
            # Final save for any remaining new channels
            if new_channels:
                self.channels.extend(new_channels)
            
        except Exception as e:
            print(f"Error in extract_all_channels: {str(e)}")
            # Try to save what we have in case of error
            if new_channels:
                self.channels.extend(new_channels)
                self.save_results(is_final=False)
    
    def save_results(self, is_final=False):
        if not self.channels:
            print("No channels to save")
            return
            
        try:
            # Sort channels by member count (descending)
            self.channels.sort(key=lambda x: x['members'], reverse=True)
            
            # Save to JSON
            with open('telegram_channels.json', 'w', encoding='utf-8') as f:
                json.dump(self.channels, f, ensure_ascii=False, indent=4)
            
            # Save to CSV
            df = pd.DataFrame(self.channels)
            # Format member numbers with commas in CSV
            df['members'] = df['members'].apply(lambda x: f"{int(x):,}" if x != 0 else "N/A")
            df.to_csv('telegram_channels.csv', index=False, encoding='utf-8-sig')
            
            # Save links of groups with >5000 members to TXT
            large_groups_5000 = [channel['link'] for channel in self.channels if channel['members'] >= 5000]
            with open('telegram_channels_5000plus.txt', 'w', encoding='utf-8') as f:
                f.write('\n'.join(large_groups_5000))
            
            if is_final:
                print(f"Final save completed. Total channels: {len(self.channels)}")
                print(f"Channels with 5000+ members: {len(large_groups_5000)}")
            else:
                print(f"Autosave completed. Current channels: {len(self.channels)}")
                print(f"Current 5000+ members: {len(large_groups_5000)}")
            
        except Exception as e:
            print(f"Error saving results: {str(e)}")
    
    def scrape(self, url):
        try:
            print(f"Starting to scrape URL: {url}")
            self.driver.get(url)
            print("Waiting for page to load...")
            
            # Wait for the main content to load
            WebDriverWait(self.driver, 30).until(
                EC.presence_of_element_located((By.CSS_SELECTOR, "div.info.flex-grow-1"))
            )
            
            time.sleep(2)
            print("Page loaded successfully")
            
            # First scroll to the end
            self.scroll_to_end()
            
            # Then extract all channels
            self.extract_all_channels()
            
            print(f"Scraping completed. Found {len(self.channels)} channels")
            
            # Final save
            self.save_results(is_final=True)
            
        except Exception as e:
            print(f"Error during scraping: {str(e)}")
            # Try to save what we have in case of error
            if self.channels:
                print("Attempting to save partial results...")
                self.save_results(is_final=True)
        finally:
            try:
                if hasattr(self, 'driver'):
                    self.driver.quit()
            except Exception as e:
                print(f"Error while closing driver: {str(e)}")

if __name__ == "__main__":
    try:
        scraper = TelegramChannelScraper()
        scraper.scrape("https://combot.org/top/telegram/groups?lng=en&page=1")
    except Exception as e:
        print(f"Fatal error: {str(e)}")
        try:
            scraper.driver.quit()
        except:
            pass

import pandas as pd

# Read the CSV file
df = pd.read_csv('telegram_channels.csv')

# Convert members column to numeric by removing commas
df['members'] = df['members'].str.replace(',', '').astype(float)

# Calculate total members
total_members = df['members'].sum()

# Format the number with commas for better readability
formatted_total = "{:,.0f}".format(total_members)

print(f"Total number of members across all channels: {formatted_total}")