Bypass CF Shield

Use cloudscraper instead of requests

1
pip3 install cloudscraper
1
2
3
import cloudscraper
scraper = cloudscraper.create_scraper()
response =scraper.get(url)

Crawl Web Dynamic Content, Need to Install Selenium

1
pip3 install selenium
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
url = "xxxx"
chrome_options = Options()
chrome_options.add_argument('--no-sandbox')  # Grant root execution permissions
chrome_options.add_argument('--headless')  # Hide browser execution
chrome_options.add_argument('--disable-dev-shm-usage') # Prevent excessive memory usage, which could cause performance issues or crashes.
chrome_options.add_argument('--user-agent=headers')  # Simulate different types of browsers or devices
chrome_options.add_argument('--disable-web-security') # Disable the browser's same-origin policy
driver = webdriver.Chrome(options=chrome_options) 
# driver = webdriver.Chrome()
driver.get(url)
driver.implicitly_wait(5)  # Implicit wait time of 5 seconds
page_content = driver.page_source  # Get the HTML source code of the current webpage
driver.quit()

Download Hanime Hentai Videos and Save in Year-Month Format

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
import os
import requests
from bs4 import BeautifulSoup
import cloudscraper
import re
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
 
 
year = 2018
for month in range(4, 13):
    base_url = "https://hanime1.me/search?query=&type=&genre=%E8%A3%8F%E7%95%AA&sort=&year={}&month={}"
    url = base_url.format(year, month)
    scraper = cloudscraper.create_scraper()
    response = scraper.get(url)
    soup = BeautifulSoup(response.text.encode("utf-8"), "html.parser")
 
    pattern = r'"(https://hanime1\.me/watch\?[^\s]+)"'  # Match links starting with "https://hanime1.me/watch?"
    matches = re.findall(pattern, str(soup))
    download_dir = f"{year}/{month:02}"
    print(download_dir)
    if not os.path.exists(download_dir):
        os.makedirs(download_dir)
 
    for matche in matches:
        clean_match = matche.strip("")  # Remove double quotes
        headers = {
            'User-Agent': ''
        }
        cookies = {
            "cookies1": ""}
        response2 = scraper.get(matche, cookies=cookies, headers=headers)
        chrome_options = Options()
        chrome_options.add_argument('--no-sandbox')  # Grant root execution permissions
        chrome_options.add_argument('--headless')  # Hide browser execution
        chrome_options.add_argument('--disable-dev-shm-usage')
        chrome_options.add_argument('--user-agent=headers')
 
        chrome_options.add_argument('--disable-web-security')
        driver = webdriver.Chrome(options=chrome_options)
        # driver = webdriver.Chrome()
        driver.get(matche)
        driver.implicitly_wait(5)
        page_content = driver.page_source
        driver.quit()
 
        # soup2 = BeautifulSoup(response.text.encode("utf-8"), "html.parser")
        soup2 = BeautifulSoup(page_content, 'html.parser')
 
        # Get 1080p
        source_1080 = soup2.find('source', {'size': '1080'})
        if source_1080:
            src = source_1080.get('src')
            print(f"1080p URL: {src}")
        else:
            # If no tag with size="1080" is found, look for size="720"
            source_720 = soup2.find('source', {'size': '720'})
 
            if source_720:
                src = source_720.get('src')
                print(f"720p URL: {src}")
            else:
                # If no tag with size="720" is found, look for size="480"
                source_480 = soup2.find('source', {'size': '480'})
 
                if source_480:
                    src = source_480.get('src')
                    print(f"480p URL: {src}")
                else:
                    print("No matching URL found")
 
        # pattern2 = soup2.find('source')
        # value = pattern2['src']
 
        # Download video
        pattern2 = soup2.find('input', {'id': 'video-sd'})
        value = pattern2['value'].split("?")[0]
        video_response = requests.get(src, stream=True)
        video_filename = download_dir
        filename = value.split('/')[-1]
        save_path = os.path.join(download_dir, filename)
        with open(save_path, 'wb') as video_file:
            video_file.write(video_response.content)
 
        print(f"Downloaded: {save_path}")
    print('Month {} download completed'.format(month))
print('Year {} download completed'.format(year))