github-actions[bot] commited on
Commit
2a38ff3
·
1 Parent(s): b4a1e45

Update from GitHub Actions

Browse files
Files changed (1) hide show
  1. main.py +30 -21
main.py CHANGED
@@ -8,7 +8,7 @@ import uvicorn
8
  import asyncio
9
  from urllib.parse import urlparse
10
  import time
11
- from curl_cffi import requests
12
  from dotenv import load_dotenv
13
 
14
  # 加载多个位置的.env文件
@@ -16,7 +16,7 @@ def load_env_files():
16
  # 优先尝试加载根目录下的.env
17
  if os.path.exists('.env'):
18
  load_dotenv('.env')
19
-
20
  # 然后尝试加载 /root/deploy/.scraper.env
21
  deploy_env_path = os.path.join('/root/deploy', '.scraper.env')
22
  if os.path.exists(deploy_env_path):
@@ -44,6 +44,15 @@ async def stream_generator(response):
44
  """生成流式响应的生成器函数"""
45
  for chunk in response.iter_content(chunk_size=8192):
46
  if chunk:
 
 
 
 
 
 
 
 
 
47
  yield chunk
48
  await asyncio.sleep(0.001) # 让出控制权,保持异步特性
49
 
@@ -55,7 +64,7 @@ async def make_request(method: str, **kwargs):
55
  print(f"开始使用{REQUEST_LIB}进行请求")
56
  if REQUEST_LIB == 'cloudscraper':
57
  scraper = cloudscraper.create_scraper(delay=10)
58
-
59
  # 设置代理
60
  proxy = os.environ.get('PROXY')
61
  if proxy:
@@ -63,21 +72,21 @@ async def make_request(method: str, **kwargs):
63
  'http': proxy,
64
  'https': proxy
65
  }
66
-
67
  # 根据方法发送请求
68
  return getattr(scraper, method.lower())(**kwargs)
69
  else:
70
  # 使用 curl_cffi
71
  proxy = os.environ.get('PROXY')
72
  proxies = {'http': proxy, 'https': proxy} if proxy else None
73
-
74
  # curl_cffi 的请求配置
75
  request_config = {
76
  **kwargs,
77
  'proxies': proxies,
78
  'impersonate': 'chrome110',
79
  }
80
-
81
  return requests.request(method, **request_config)
82
 
83
  @app.get("/", response_class=HTMLResponse)
@@ -106,7 +115,7 @@ async def proxy(request: Request):
106
  detail="未提供有效的x-ip-token header",
107
  headers={"WWW-Authenticate": "x-ip-token"}
108
  )
109
-
110
  # 验证token
111
  if auth_header != env_token:
112
  raise HTTPException(
@@ -118,26 +127,26 @@ async def proxy(request: Request):
118
  target_url = request.query_params.get("url")
119
  if not target_url:
120
  raise HTTPException(status_code=400, detail="必须提供目标URL")
121
-
122
  # 获取home_url
123
  home_url = request.query_params.get("home")
124
  if not home_url:
125
  # 从target_url中提取home_url
126
  parsed_url = urlparse(target_url)
127
  home_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
128
-
129
  # 检查是否请求流式响应
130
  stream_request = "stream" in request.query_params and request.query_params["stream"].lower() in ["true", "1", "yes"]
131
-
132
  # 获取请求体
133
  body = await request.body()
134
-
135
  # 获取查询参数
136
  params = dict(request.query_params)
137
  # 从查询参数中移除url和stream参数
138
  params.pop("url", None)
139
  params.pop("stream", None)
140
-
141
  # 获取原始请求头
142
  headers = dict(request.headers)
143
  # 移除可能导致问题的头
@@ -151,7 +160,7 @@ async def proxy(request: Request):
151
  headers.pop("host", None)
152
  headers.pop("referer", None)
153
  print(f"{headers}")
154
-
155
  # 构建请求参数
156
  request_kwargs = {
157
  "url": target_url,
@@ -159,14 +168,14 @@ async def proxy(request: Request):
159
  "params": params,
160
  "stream": stream_request # 设置stream参数
161
  }
162
-
163
  # 如果有请求体,添加到请求参数中
164
  if body:
165
  request_kwargs["data"] = body
166
-
167
  # 使用统一的请求函数发送请求
168
  response = await make_request(request.method, **request_kwargs)
169
-
170
  # 处理流式响应
171
  if stream_request:
172
  # 创建响应头字典
@@ -174,7 +183,7 @@ async def proxy(request: Request):
174
  for header_name, header_value in response.headers.items():
175
  if header_name.lower() not in ('content-encoding', 'transfer-encoding', 'content-length'):
176
  headers_dict[header_name] = header_value
177
-
178
  # 返回流式响应
179
  return StreamingResponse(
180
  stream_generator(response),
@@ -188,18 +197,18 @@ async def proxy(request: Request):
188
  content=response.content,
189
  status_code=response.status_code,
190
  )
191
-
192
  # 转发响应头
193
  for header_name, header_value in response.headers.items():
194
  if header_name.lower() not in ('content-encoding', 'transfer-encoding', 'content-length'):
195
  proxy_response.headers[header_name] = header_value
196
-
197
  # 转发cookies
198
  for cookie_name, cookie_value in response.cookies.items():
199
  proxy_response.set_cookie(key=cookie_name, value=cookie_value)
200
-
201
  return proxy_response
202
-
203
  except Exception as e:
204
  error = f"代理请求失败: {str(e)}"
205
  print(error)
 
8
  import asyncio
9
  from urllib.parse import urlparse
10
  import time
11
+ from curl_cffi import requests
12
  from dotenv import load_dotenv
13
 
14
  # 加载多个位置的.env文件
 
16
  # 优先尝试加载根目录下的.env
17
  if os.path.exists('.env'):
18
  load_dotenv('.env')
19
+
20
  # 然后尝试加载 /root/deploy/.scraper.env
21
  deploy_env_path = os.path.join('/root/deploy', '.scraper.env')
22
  if os.path.exists(deploy_env_path):
 
44
  """生成流式响应的生成器函数"""
45
  for chunk in response.iter_content(chunk_size=8192):
46
  if chunk:
47
+ # 打印收到的流内容
48
+ try:
49
+ # 尝试解码为文本并打印
50
+ decoded_chunk = chunk.decode('utf-8', errors='replace')
51
+ print(f"收到流数据 [大小: {len(chunk)} 字节]: {decoded_chunk}")
52
+ except Exception as e:
53
+ # 如果解码失败,打印二进制数据的长度和错误
54
+ print(f"收到流数据 [大小: {len(chunk)} 字节]: (无法解码为文本: {str(e)})")
55
+
56
  yield chunk
57
  await asyncio.sleep(0.001) # 让出控制权,保持异步特性
58
 
 
64
  print(f"开始使用{REQUEST_LIB}进行请求")
65
  if REQUEST_LIB == 'cloudscraper':
66
  scraper = cloudscraper.create_scraper(delay=10)
67
+
68
  # 设置代理
69
  proxy = os.environ.get('PROXY')
70
  if proxy:
 
72
  'http': proxy,
73
  'https': proxy
74
  }
75
+
76
  # 根据方法发送请求
77
  return getattr(scraper, method.lower())(**kwargs)
78
  else:
79
  # 使用 curl_cffi
80
  proxy = os.environ.get('PROXY')
81
  proxies = {'http': proxy, 'https': proxy} if proxy else None
82
+
83
  # curl_cffi 的请求配置
84
  request_config = {
85
  **kwargs,
86
  'proxies': proxies,
87
  'impersonate': 'chrome110',
88
  }
89
+
90
  return requests.request(method, **request_config)
91
 
92
  @app.get("/", response_class=HTMLResponse)
 
115
  detail="未提供有效的x-ip-token header",
116
  headers={"WWW-Authenticate": "x-ip-token"}
117
  )
118
+
119
  # 验证token
120
  if auth_header != env_token:
121
  raise HTTPException(
 
127
  target_url = request.query_params.get("url")
128
  if not target_url:
129
  raise HTTPException(status_code=400, detail="必须提供目标URL")
130
+
131
  # 获取home_url
132
  home_url = request.query_params.get("home")
133
  if not home_url:
134
  # 从target_url中提取home_url
135
  parsed_url = urlparse(target_url)
136
  home_url = f"{parsed_url.scheme}://{parsed_url.netloc}/"
137
+
138
  # 检查是否请求流式响应
139
  stream_request = "stream" in request.query_params and request.query_params["stream"].lower() in ["true", "1", "yes"]
140
+
141
  # 获取请求体
142
  body = await request.body()
143
+
144
  # 获取查询参数
145
  params = dict(request.query_params)
146
  # 从查询参数中移除url和stream参数
147
  params.pop("url", None)
148
  params.pop("stream", None)
149
+
150
  # 获取原始请求头
151
  headers = dict(request.headers)
152
  # 移除可能导致问题的头
 
160
  headers.pop("host", None)
161
  headers.pop("referer", None)
162
  print(f"{headers}")
163
+
164
  # 构建请求参数
165
  request_kwargs = {
166
  "url": target_url,
 
168
  "params": params,
169
  "stream": stream_request # 设置stream参数
170
  }
171
+
172
  # 如果有请求体,添加到请求参数中
173
  if body:
174
  request_kwargs["data"] = body
175
+
176
  # 使用统一的请求函数发送请求
177
  response = await make_request(request.method, **request_kwargs)
178
+
179
  # 处理流式响应
180
  if stream_request:
181
  # 创建响应头字典
 
183
  for header_name, header_value in response.headers.items():
184
  if header_name.lower() not in ('content-encoding', 'transfer-encoding', 'content-length'):
185
  headers_dict[header_name] = header_value
186
+
187
  # 返回流式响应
188
  return StreamingResponse(
189
  stream_generator(response),
 
197
  content=response.content,
198
  status_code=response.status_code,
199
  )
200
+
201
  # 转发响应头
202
  for header_name, header_value in response.headers.items():
203
  if header_name.lower() not in ('content-encoding', 'transfer-encoding', 'content-length'):
204
  proxy_response.headers[header_name] = header_value
205
+
206
  # 转发cookies
207
  for cookie_name, cookie_value in response.cookies.items():
208
  proxy_response.set_cookie(key=cookie_name, value=cookie_value)
209
+
210
  return proxy_response
211
+
212
  except Exception as e:
213
  error = f"代理请求失败: {str(e)}"
214
  print(error)