1
2
~$: pip install requests

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
# -*- coding: utf-8 -*-
"""
@file: httpkit
@author: MR.N
@created: 2022/4/2 4月
@updated: 2022/5/23 5月
@version: 1.0
@blog: https://blog.csdn.net/qq_21264377
"""


import time
import urllib.parse
import urllib.request
import requests
import urllib3
import http.cookiejar
import ssl
import socket
import gzip
from uas import *
import random


SOCKET_TIMEOUT = 30
HTTPS_TIMEOUT = 10


# ...(略)


def request_res(remote_task=None, ret=[], dtype=0, max_retry=3):
    if remote_task is None:
        ret += ['', '', '', -1]
        return 'err'
    if not isinstance(remote_task, RemoteTask):
        ret += ['', '', '', -1]
        return 'err'
    url = remote_task.url
    if not valid_https(url):
        ret += ['', '', '', -1]
        return 'err'
    referer = remote_task.referer
    cookies = remote_task.cookies
    ua = remote_task.ua
    if ua is None:
        ua = unspecific_ua()
    headers = {
        'User-Agent': ua,
        # 'Accept-Encoding': 'gzip, deflate, br',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Pragma': 'no-cache',
        'Cache-Control': 'no-cache',
        'Sec-Fetch-User': '?1',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Site': 'none',
        'Upgrade-Insecure-Requests': '1',
        'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
        'Connection': 'keep-alive',
    }
    if referer is None:
        headers['Referer'] = url
    else:
        headers['Referer'] = referer
    if cookies is not None:
        headers['Cookie'] = cookies
    headers['host'] = url.split('/')[2]
    # print(headers)
    socket.setdefaulttimeout(SOCKET_TIMEOUT)
    ssl._create_default_https_context = ssl._create_unverified_context
    session = requests.Session()
    attempts = 0
    status_code = -1
    response = None
    while attempts < max_retry and status_code != 200:
        attempts += 1
        try:
            response = session.request(method='GET', url=url, headers=headers, timeout=HTTPS_TIMEOUT)
            status_code = response.status_code
        except TimeoutError:
            status_code = 404
        except requests.exceptions.Timeout:
            status_code = 404
        except requests.exceptions.ReadTimeout:
            status_code = 404
        finally:
            if status_code != 200:
                time.sleep(.11)
    if response is not None and response.status_code == 200:
        data = response.content
        content_encoding = response.headers.get('Content-Encoding')
        if content_encoding is not None and content_encoding.strip() != '' and content_encoding.lower() in ['gzip',
                                                                                                            'deflate']:
            # data = gzip.decompress(data)
            pass
        content_type = response.headers.get('Content-Type')
        if content_type is not None and 'charset=' in content_type:
            encoding = content_type.split(';')[-1].split('=')[-1]
        else:
            encoding = response.apparent_encoding
        if encoding is not None and encoding.strip() != '':
            # print(encoding)
            data = data.decode(encoding=encoding)
        else:
            data = data.decode('UTF-8')
        # print(content_encoding, len(data))
        cookies = response.cookies
        cookie_res = ''
        for cookie in cookies:
            cookie_res += cookie.name + '=' + cookie.value + ';'
        ret += [data, url, cookie_res, status_code]
        if session is not None:
            session.close()
        if response is not None:
            response.close()
        return 'success'
    else:
        if session is not None:
            session.close()
        if response is not None:
            response.close()
        ret += ['', '', '', status_code]
        return 'failure'