elements that match class conditions
+ div_elements = soup.find_all('div', class_="panel-row ng-scope")
+
+ # Variable to determine if loop should continue
+ should_continue = False
+
+ # Iterate through the div elements to find links and dates
+ for div in div_elements:
+ date_span = div.find('span', class_='date ng-binding')
+ if date_span:
+ date_text = date_span.text.strip()
+ if is_date_valid(date_text):
+ should_continue = True
+ link = div.find('a', href=True, attrs={'ng-bind-html': 'x.docSubtitle|trustHtml'})
+ if link and "处罚信息公开表" in link.text:
+ href = link['href']
+ full_url = base_url + href
+ if "//cn/view/pages/" not in full_url:
+ unique_urls.add(full_url)
+ else:
+ # Since this date is invalid and dates are sorted in descending order, no need to continue
+ should_continue = False
+ break
+
+ # Save URLs if they exceed 2000 and reset unique_urls
+ if len(unique_urls) >= 10:
+ save_urls_to_file(unique_urls, file_index)
+ unique_urls.clear()
+
+ # If the current file exceeds 20000 URLs, start a new file
+ if urls_in_current_file >= 20:
+ file_index += 1
+ urls_in_current_file = 0
+
+ # Check if loop should continue
+ if not should_continue:
+ break
+
+ # Try to find and click the next page button
+ try:
+ next_button = WebDriverWait(driver, 10).until(
+ EC.element_to_be_clickable((By.XPATH, "//a[@ng-click='pager.next()']"))
+ )
+ ActionChains(driver).move_to_element(next_button).click().perform()
+ random_wait() # Wait for the next page to load
+ except Exception as e:
+ print("No more pages or error occurred:", e)
+ break
+
+ # Save remaining URLs if any
+ if unique_urls:
+ save_urls_to_file(unique_urls, file_index)
+
+ # Close the browser
+ driver.quit()
+
+ print("Total URLs saved:", total_urls_saved)
+
+
+# Example usage
+base_page_url = "https://www.cbirc.gov.cn/cn/view/pages/ItemList.html?itemPId=923&itemId=4115&itemUrl=ItemListRightList.html&itemName=%E7%9B%91%E7%AE%A1%E5%88%86%E5%B1%80%E6%9C%AC%E7%BA%A7&itemsubPId=931&itemsubPName=%E8%A1%8C%E6%94%BF%E5%A4%84%E7%BD%9A#382"
base_url = 'https://www.cbirc.gov.cn/cn/view/pages/'
-
-# Set to store unique URLs
-unique_urls = set()
-
-
-# Function to check date
-def is_date_valid(date_text):
- given_date = datetime.strptime(date_text, "%Y-%m-%d")
- return given_date >= datetime(2023, 11, 28)
-
-
-# Visit the initial page
-driver.get(base_page_url)
-cur_page = 0
-
-# Keep processing until a date before June 1, 2023, is found
-while True:
- cur_page += 1
- print("Visiting new page:" + str(cur_page))
-
- # Wait for JavaScript to load
- random_wait()
-
- # Get the page source after JS execution
- html = driver.page_source
-
- # Parse the HTML using BeautifulSoup
- soup = BeautifulSoup(html, 'html.parser')
-
- # Find all
elements that match class conditions
- div_elements = soup.find_all('div', class_="panel-row ng-scope")
-
- # Variable to determine if loop should continue
- should_continue = False
-
- # Iterate through the div elements to find links and dates
- for div in div_elements:
- date_span = div.find('span', class_='date ng-binding')
- if date_span:
- date_text = date_span.text.strip()
- if is_date_valid(date_text):
- should_continue = True
- link = div.find('a', href=True, attrs={'ng-bind-html': 'x.docSubtitle|trustHtml'})
- if link and "处罚信息公开表" in link.text:
- href = link['href']
- full_url = base_url + href
- if "//cn/view/pages/" not in full_url:
- unique_urls.add(full_url)
- else:
- # Since this date is invalid and dates are sorted in descending order, no need to continue
- should_continue = False
- break
-
- # Check if loop should continue
- if not should_continue:
- break
-
- # Try to find and click the next page button
- try:
- next_button = WebDriverWait(driver, 10).until(
- EC.element_to_be_clickable((By.XPATH, "//a[@ng-click='pager.next()']"))
- )
- ActionChains(driver).move_to_element(next_button).click().perform()
- random_wait() # Wait for the next page to load
- except Exception as e:
- print("No more pages or error occurred:", e)
- break
-
-# Close the browser
-driver.quit()
-
-# Print all unique URLs and count them
-cnt = 0
-
-# Open a file to write
-with open('url2.txt', 'w') as file:
- for url in unique_urls:
- cnt += 1
- file.write(url + '\n') # Write each URL followed by a newline
-
-print("URLs have been saved to urls1.txt")
-print("Total URLs found:", cnt)
+date_limit = datetime(2024, 7, 24)
+output_path_prefix = 'url'
+fetch_unique_urls(base_page_url, base_url, date_limit, output_path_prefix)