#!/usr/bin/env python3 """ FormDS Web Scraper - Simple Version (Basic Fields Only) Scrapes FormDS.com for current date SEC Form D filings and stores them in MySQL. Only extracts: company, reported_funding, incremental_cash, filing_date, new_or_amended Author: HemantaBhusal Version: 7.1 (Simple) """ import os import sys import urllib.request import urllib.error from html.parser import HTMLParser import mysql.connector import time import logging from datetime import datetime sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from global_config import db_config logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s') class TableParser(HTMLParser): """Custom HTML parser to extract table data""" def __init__(self): super().__init__() self.in_table = False self.in_row = False self.in_cell = False self.current_row = [] self.current_cell = "" self.rows = [] self.cell_count = 0 def handle_starttag(self, tag, attrs): if tag == 'table': self.in_table = True elif tag == 'tr' and self.in_table: self.in_row = True self.current_row = [] self.cell_count = 0 elif tag == 'td' and self.in_row: self.in_cell = True self.current_cell = "" self.cell_count += 1 def handle_endtag(self, tag): if tag == 'table': self.in_table = False elif tag == 'tr' and self.in_row: self.rows.append(self.current_row) self.in_row = False elif tag == 'td' and self.in_cell: self.current_row.append(self.current_cell.strip()) self.in_cell = False def handle_data(self, data): if self.in_cell: self.current_cell += data class FormDSScraper: def __init__(self): self.base_url = "https://www.formds.com/filings/newest" os.environ['TZ'] = 'America/New_York' time.tzset() # apply the timezone # Store current date (New York) self.current_date = datetime.now().strftime('%Y-%m-%d') #self.current_date = '2025-09-16' logging.info(f"Simple scraper initialized for {self.current_date}") def make_request(self, url): """Make HTTP request using urllib""" try: req = urllib.request.Request( url, headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Connection': 'keep-alive' } ) with urllib.request.urlopen(req, timeout=10) as response: return response.read().decode('utf-8') except urllib.error.URLError as e: logging.error(f"Failed to fetch {url}: {e}") return None except Exception as e: logging.error(f"Unexpected error fetching {url}: {e}") return None def get_page(self, page_num=1): """Fetch page from FormDS and parse HTML""" try: url = f"{self.base_url}?page={page_num}" html_content = self.make_request(url) if not html_content: return None parser = TableParser() parser.feed(html_content) return parser.rows except Exception as e: logging.error(f"Failed to parse page {page_num}: {e}") return None def assign_order_numbers(self, scraped_data): """Assign order numbers with highest number for first record""" total_records = len(scraped_data) for i, filing in enumerate(scraped_data): filing['order_num'] = total_records - i # First record gets highest number def scrape_pages(self, max_pages=20): """Main scraping - collect all current date filings""" page_num = 1 scraped_data = [] logging.info(f"Starting scraping for {self.current_date}...") while page_num <= max_pages: rows = self.get_page(page_num) if not rows: break page_data = [] for row_idx, row in enumerate(rows[1:], 1): # Skip header if len(row) >= 6: try: company = row[0] filing_date = row[4] # Stop if different date found if filing_date != self.current_date: if page_data: scraped_data.extend(page_data) # Assign order numbers before returning self.assign_order_numbers(scraped_data) logging.info(f"Date changed to {filing_date} - returning {len(scraped_data)} filings") return scraped_data # Clean company name - remove newlines and extra whitespace company = ' '.join(company.split()) # Collect basic filing data (without order_num for now) filing_data = { 'company': company, 'reported_funding': row[2], 'incremental_cash': row[3], 'filing_date': filing_date, 'new_or_amended': row[5] } page_data.append(filing_data) except Exception as e: logging.warning(f"Error parsing row {row_idx}: {e}") continue # Add page data if page_data: scraped_data.extend(page_data) logging.info(f"Page {page_num}: Found {len(page_data)} filings") else: logging.info(f"No current date data on page {page_num}") break page_num += 1 time.sleep(0.5) # Small delay between pages # Assign order numbers before returning (for normal completion) self.assign_order_numbers(scraped_data) logging.info(f"Scraping completed - found {len(scraped_data)} total filings") return scraped_data def save_to_mysql(self, data): """Delete all data and save new data""" if not data: print("No data to save") return connection = None try: connection = mysql.connector.connect(**db_config) cursor = connection.cursor() # Delete all existing data cursor.execute("TRUNCATE TABLE filings1") logging.info("Cleared all existing data from table") insert_query = """ INSERT INTO filings1 (order_num, company, reported_funding, incremental_cash, filing_date, new_or_amended) VALUES (%s, %s, %s, %s, %s, %s) """ # Prepare batch data batch_data = [] for filing in data: batch_data.append(( filing['order_num'], filing['company'], filing['reported_funding'], filing['incremental_cash'], filing['filing_date'], filing['new_or_amended'] )) # Execute batch insert cursor.executemany(insert_query, batch_data) connection.commit() print(f"Successfully saved {len(data)} records (fresh data)") except mysql.connector.Error as error: print(f"MySQL Error: {error}") logging.error(f"MySQL database error during batch save: {error}") if connection: connection.rollback() except Exception as e: print(f"Save error: {e}") logging.error(f"Unexpected error during batch save: {e}") if connection: connection.rollback() finally: if connection and connection.is_connected(): cursor.close() connection.close() def main(): """Main execution""" try: # Run simple scraper scraper = FormDSScraper() scraped_data = scraper.scrape_pages(max_pages=25) if scraped_data and len(scraped_data) > 0: scraper.save_to_mysql(scraped_data) else: print("No new filings found") except Exception as e: print(f"Scraper failed: {e}") logging.error(f"Fatal error: {e}") if __name__ == "__main__": main()