Spaces:
Paused
Paused
Upload 16 files
Browse files- .env.example +2 -2
- src/ProxyPool.js +34 -52
- src/ProxyServer.js +204 -168
- src/lightweight-client-express.js +1 -1
- src/lightweight-client.js +20 -38
- src/proxy/chrome_proxy_server_android_arm64 +2 -2
- src/proxy/chrome_proxy_server_linux_amd64 +2 -2
- src/proxy/chrome_proxy_server_windows_amd64.exe +2 -2
.env.example
CHANGED
|
@@ -6,8 +6,8 @@ NOTION_COOKIE="cookie1|cookie2"
|
|
| 6 |
PROXY_AUTH_TOKEN="sk-text"
|
| 7 |
# (可选)代理地址
|
| 8 |
PROXY_URL=""
|
| 9 |
-
#
|
| 10 |
-
USE_NATIVE_PROXY_POOL=
|
| 11 |
|
| 12 |
# (可选)代理国家,支持us uk jp de fr ca
|
| 13 |
PROXY_COUNTRY="us"
|
|
|
|
| 6 |
PROXY_AUTH_TOKEN="sk-text"
|
| 7 |
# (可选)代理地址
|
| 8 |
PROXY_URL=""
|
| 9 |
+
# (可选)是否使用本地代理池,启用前确保没有设置其它代理,否则速度会减慢或者出现其它问题,推荐启用
|
| 10 |
+
USE_NATIVE_PROXY_POOL=true
|
| 11 |
|
| 12 |
# (可选)代理国家,支持us uk jp de fr ca
|
| 13 |
PROXY_COUNTRY="us"
|
src/ProxyPool.js
CHANGED
|
@@ -9,15 +9,15 @@ class ProxyPool {
|
|
| 9 |
* @param {Object} options - 配置选项
|
| 10 |
* @param {number} options.targetCount - 目标代理数量,默认20
|
| 11 |
* @param {number} options.batchSize - 每次获取的代理数量,默认20
|
| 12 |
-
* @param {number} options.testTimeout - 测试代理超时时间(毫秒),默认
|
| 13 |
-
* @param {number} options.requestTimeout - 请求目标网站超时时间(毫秒),默认
|
| 14 |
* @param {string} options.targetUrl - 目标网站URL,默认'https://www.notion.so'
|
| 15 |
-
* @param {number} options.concurrentRequests - 并发请求数量,默认
|
| 16 |
* @param {number} options.minThreshold - 可用代理数量低于此阈值时自动补充,默认5
|
| 17 |
* @param {number} options.checkInterval - 检查代理池状态的时间间隔(毫秒),默认30000
|
| 18 |
* @param {string} options.proxyProtocol - 代理协议,默认'http'
|
| 19 |
-
* @param {number} options.maxRefillAttempts - 最大补充尝试次数,默认
|
| 20 |
-
* @param {number} options.retryDelay - 重试延迟(毫秒),默认
|
| 21 |
* @param {boolean} options.useCache - 是否使用缓存,默认true
|
| 22 |
* @param {number} options.cacheExpiry - 缓存过期时间(毫秒),默认3600000 (1小时)
|
| 23 |
* @param {string} options.logLevel - 日志级别,可选值:'debug', 'info', 'warn', 'error', 'none',默认'info'
|
|
@@ -27,15 +27,15 @@ class ProxyPool {
|
|
| 27 |
// 配置参数
|
| 28 |
this.targetCount = options.targetCount || 20;
|
| 29 |
this.batchSize = options.batchSize || 20;
|
| 30 |
-
this.testTimeout = options.testTimeout ||
|
| 31 |
-
this.requestTimeout = options.requestTimeout ||
|
| 32 |
this.targetUrl = options.targetUrl || 'https://www.notion.so';
|
| 33 |
-
this.concurrentRequests = options.concurrentRequests ||
|
| 34 |
this.minThreshold = options.minThreshold || 5;
|
| 35 |
this.checkInterval = options.checkInterval || 30000; // 默认30秒检查一次
|
| 36 |
this.proxyProtocol = options.proxyProtocol || 'http';
|
| 37 |
-
this.maxRefillAttempts = options.maxRefillAttempts ||
|
| 38 |
-
this.retryDelay = options.retryDelay ||
|
| 39 |
this.useCache = options.useCache !== undefined ? options.useCache : true;
|
| 40 |
this.cacheExpiry = options.cacheExpiry || 3600000; // 默认1小时
|
| 41 |
this.logLevel = options.logLevel || 'info'; // 默认日志级别为info
|
|
@@ -237,7 +237,7 @@ class ProxyPool {
|
|
| 237 |
|
| 238 |
// 计算本次需要获取的批次大小
|
| 239 |
const remainingNeeded = this.targetCount - this.availableProxies.length;
|
| 240 |
-
const batchSizeNeeded =
|
| 241 |
|
| 242 |
// 获取代理
|
| 243 |
const proxies = await this.getProxiesFromProvider(batchSizeNeeded);
|
|
@@ -253,7 +253,6 @@ class ProxyPool {
|
|
| 253 |
|
| 254 |
if (newProxies.length === 0) {
|
| 255 |
this.log('debug', '所有获取的代理都已存在,继续获取新代理...');
|
| 256 |
-
// 减少等待时间,立即继续尝试
|
| 257 |
continue;
|
| 258 |
}
|
| 259 |
|
|
@@ -275,9 +274,9 @@ class ProxyPool {
|
|
| 275 |
break;
|
| 276 |
}
|
| 277 |
|
| 278 |
-
//
|
| 279 |
if (this.availableProxies.length < this.targetCount) {
|
| 280 |
-
await new Promise(resolve => setTimeout(resolve,
|
| 281 |
}
|
| 282 |
}
|
| 283 |
} catch (error) {
|
|
@@ -427,36 +426,18 @@ class ProxyPool {
|
|
| 427 |
async getProxiesFromProvider(count = null) {
|
| 428 |
try {
|
| 429 |
const requestCount = count || this.batchSize;
|
| 430 |
-
//
|
| 431 |
const actualCount = Math.min(requestCount, 10);
|
|
|
|
|
|
|
| 432 |
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
this.log('debug', `需要 ${requestCount} 个代理,将发送 ${actualRequests} 个并行请求`);
|
| 439 |
-
|
| 440 |
-
// 并行发送多个请求
|
| 441 |
-
const requestPromises = [];
|
| 442 |
-
for (let i = 0; i < actualRequests; i++) {
|
| 443 |
-
const url = `https://proxy.doudouzi.me/random/${this.proxyCountry}?number=${actualCount}&protocol=${this.proxyProtocol}&type=json`;
|
| 444 |
-
requestPromises.push(
|
| 445 |
-
axios.get(url, {
|
| 446 |
-
timeout: 10000,
|
| 447 |
-
validateStatus: status => true
|
| 448 |
-
})
|
| 449 |
-
);
|
| 450 |
-
}
|
| 451 |
-
|
| 452 |
-
// 等待所有请求完成
|
| 453 |
-
const responses = await Promise.all(requestPromises);
|
| 454 |
-
|
| 455 |
-
// 处理所有响应,合并代理列表
|
| 456 |
-
let proxies = [];
|
| 457 |
|
| 458 |
-
|
| 459 |
-
|
| 460 |
|
| 461 |
// 处理不同的返回格式
|
| 462 |
if (typeof response.data === 'string') {
|
|
@@ -502,10 +483,13 @@ class ProxyPool {
|
|
| 502 |
}
|
| 503 |
}
|
| 504 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 505 |
}
|
| 506 |
-
|
| 507 |
-
this.log('debug', `成功获取 ${proxies.length} 个代理`);
|
| 508 |
-
return proxies;
|
| 509 |
} catch (error) {
|
| 510 |
this.log('error', '获取代理出错:', error.message);
|
| 511 |
return [];
|
|
@@ -522,7 +506,7 @@ class ProxyPool {
|
|
| 522 |
const remainingNeeded = this.targetCount - this.availableProxies.length;
|
| 523 |
|
| 524 |
// 增加并发数以加快处理速度
|
| 525 |
-
const concurrentRequests = Math.min(this.concurrentRequests *
|
| 526 |
|
| 527 |
// 分批处理代理
|
| 528 |
for (let i = 0; i < proxies.length; i += concurrentRequests) {
|
|
@@ -602,9 +586,9 @@ class ProxyPool {
|
|
| 602 |
'Connection': 'keep-alive',
|
| 603 |
'Upgrade-Insecure-Requests': '1'
|
| 604 |
},
|
| 605 |
-
timeout:
|
| 606 |
validateStatus: status => true,
|
| 607 |
-
maxRedirects:
|
| 608 |
followRedirect: true
|
| 609 |
});
|
| 610 |
|
|
@@ -738,12 +722,10 @@ async function example() {
|
|
| 738 |
minThreshold: 3, // 当可用代理少于3个时,自动补充
|
| 739 |
checkInterval: 60000, // 每60秒检查一次
|
| 740 |
targetUrl: 'https://www.notion.so',
|
| 741 |
-
concurrentRequests:
|
| 742 |
useCache: true, // 启用缓存
|
| 743 |
-
maxRefillAttempts:
|
| 744 |
-
retryDelay:
|
| 745 |
-
testTimeout: 3000, // 减少测试超时时间
|
| 746 |
-
requestTimeout: 5000, // 减少请求超时时间
|
| 747 |
logLevel: 'info', // 设置日志级别
|
| 748 |
showProgressBar: true // 启用进度条
|
| 749 |
});
|
|
|
|
| 9 |
* @param {Object} options - 配置选项
|
| 10 |
* @param {number} options.targetCount - 目标代理数量,默认20
|
| 11 |
* @param {number} options.batchSize - 每次获取的代理数量,默认20
|
| 12 |
+
* @param {number} options.testTimeout - 测试代理超时时间(毫秒),默认5000
|
| 13 |
+
* @param {number} options.requestTimeout - 请求目标网站超时时间(毫秒),默认10000
|
| 14 |
* @param {string} options.targetUrl - 目标网站URL,默认'https://www.notion.so'
|
| 15 |
+
* @param {number} options.concurrentRequests - 并发请求数量,默认10
|
| 16 |
* @param {number} options.minThreshold - 可用代理数量低于此阈值时自动补充,默认5
|
| 17 |
* @param {number} options.checkInterval - 检查代理池状态的时间间隔(毫秒),默认30000
|
| 18 |
* @param {string} options.proxyProtocol - 代理协议,默认'http'
|
| 19 |
+
* @param {number} options.maxRefillAttempts - 最大补充尝试次数,默认20
|
| 20 |
+
* @param {number} options.retryDelay - 重试延迟(毫秒),默认1000
|
| 21 |
* @param {boolean} options.useCache - 是否使用缓存,默认true
|
| 22 |
* @param {number} options.cacheExpiry - 缓存过期时间(毫秒),默认3600000 (1小时)
|
| 23 |
* @param {string} options.logLevel - 日志级别,可选值:'debug', 'info', 'warn', 'error', 'none',默认'info'
|
|
|
|
| 27 |
// 配置参数
|
| 28 |
this.targetCount = options.targetCount || 20;
|
| 29 |
this.batchSize = options.batchSize || 20;
|
| 30 |
+
this.testTimeout = options.testTimeout || 5000;
|
| 31 |
+
this.requestTimeout = options.requestTimeout || 10000;
|
| 32 |
this.targetUrl = options.targetUrl || 'https://www.notion.so';
|
| 33 |
+
this.concurrentRequests = options.concurrentRequests || 10;
|
| 34 |
this.minThreshold = options.minThreshold || 5;
|
| 35 |
this.checkInterval = options.checkInterval || 30000; // 默认30秒检查一次
|
| 36 |
this.proxyProtocol = options.proxyProtocol || 'http';
|
| 37 |
+
this.maxRefillAttempts = options.maxRefillAttempts || 30; // 减少最大尝试次数
|
| 38 |
+
this.retryDelay = options.retryDelay || 1000; // 减少重试延迟
|
| 39 |
this.useCache = options.useCache !== undefined ? options.useCache : true;
|
| 40 |
this.cacheExpiry = options.cacheExpiry || 3600000; // 默认1小时
|
| 41 |
this.logLevel = options.logLevel || 'info'; // 默认日志级别为info
|
|
|
|
| 237 |
|
| 238 |
// 计算本次需要获取的批次大小
|
| 239 |
const remainingNeeded = this.targetCount - this.availableProxies.length;
|
| 240 |
+
const batchSizeNeeded = remainingNeeded; // 获取更多代理以提高成功率
|
| 241 |
|
| 242 |
// 获取代理
|
| 243 |
const proxies = await this.getProxiesFromProvider(batchSizeNeeded);
|
|
|
|
| 253 |
|
| 254 |
if (newProxies.length === 0) {
|
| 255 |
this.log('debug', '所有获取的代理都已存在,继续获取新代理...');
|
|
|
|
| 256 |
continue;
|
| 257 |
}
|
| 258 |
|
|
|
|
| 274 |
break;
|
| 275 |
}
|
| 276 |
|
| 277 |
+
// 如果还没补充到足够的代理,等待一段时间再继续
|
| 278 |
if (this.availableProxies.length < this.targetCount) {
|
| 279 |
+
await new Promise(resolve => setTimeout(resolve, this.retryDelay));
|
| 280 |
}
|
| 281 |
}
|
| 282 |
} catch (error) {
|
|
|
|
| 426 |
async getProxiesFromProvider(count = null) {
|
| 427 |
try {
|
| 428 |
const requestCount = count || this.batchSize;
|
| 429 |
+
// 限制请求数量最大为10
|
| 430 |
const actualCount = Math.min(requestCount, 10);
|
| 431 |
+
const url = `https://proxy.doudouzi.me/random/${this.proxyCountry}?number=${actualCount}&protocol=${this.proxyProtocol}&type=json`;
|
| 432 |
+
this.log('debug', `正在获取代理,URL: ${url}`);
|
| 433 |
|
| 434 |
+
const response = await axios.get(url, {
|
| 435 |
+
timeout: 10000,
|
| 436 |
+
validateStatus: status => true
|
| 437 |
+
});
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
|
| 439 |
+
if (response.data) {
|
| 440 |
+
let proxies = [];
|
| 441 |
|
| 442 |
// 处理不同的返回格式
|
| 443 |
if (typeof response.data === 'string') {
|
|
|
|
| 483 |
}
|
| 484 |
}
|
| 485 |
}
|
| 486 |
+
|
| 487 |
+
this.log('debug', `成功获取 ${proxies.length} 个代理`);
|
| 488 |
+
return proxies;
|
| 489 |
+
} else {
|
| 490 |
+
this.log('error', '获取代理失败: 返回数据格式不正确');
|
| 491 |
+
return [];
|
| 492 |
}
|
|
|
|
|
|
|
|
|
|
| 493 |
} catch (error) {
|
| 494 |
this.log('error', '获取代理出错:', error.message);
|
| 495 |
return [];
|
|
|
|
| 506 |
const remainingNeeded = this.targetCount - this.availableProxies.length;
|
| 507 |
|
| 508 |
// 增加并发数以加快处理速度
|
| 509 |
+
const concurrentRequests = Math.min(this.concurrentRequests * 2, 20);
|
| 510 |
|
| 511 |
// 分批处理代理
|
| 512 |
for (let i = 0; i < proxies.length; i += concurrentRequests) {
|
|
|
|
| 586 |
'Connection': 'keep-alive',
|
| 587 |
'Upgrade-Insecure-Requests': '1'
|
| 588 |
},
|
| 589 |
+
timeout: this.requestTimeout,
|
| 590 |
validateStatus: status => true,
|
| 591 |
+
maxRedirects: 10,
|
| 592 |
followRedirect: true
|
| 593 |
});
|
| 594 |
|
|
|
|
| 722 |
minThreshold: 3, // 当可用代理少于3个时,自动补充
|
| 723 |
checkInterval: 60000, // 每60秒检查一次
|
| 724 |
targetUrl: 'https://www.notion.so',
|
| 725 |
+
concurrentRequests: 15, // 增加并发请求数
|
| 726 |
useCache: true, // 启用缓存
|
| 727 |
+
maxRefillAttempts: 15, // 减少最大尝试次数
|
| 728 |
+
retryDelay: 1000, // 减少重试延迟
|
|
|
|
|
|
|
| 729 |
logLevel: 'info', // 设置日志级别
|
| 730 |
showProgressBar: true // 启用进度条
|
| 731 |
});
|
src/ProxyServer.js
CHANGED
|
@@ -1,169 +1,205 @@
|
|
| 1 |
-
import { spawn } from 'child_process';
|
| 2 |
-
import { fileURLToPath } from 'url';
|
| 3 |
-
import { dirname, join } from 'path';
|
| 4 |
-
import fs from 'fs';
|
| 5 |
-
import os from 'os';
|
| 6 |
-
import dotenv from 'dotenv';
|
| 7 |
-
import chalk from 'chalk';
|
| 8 |
-
|
| 9 |
-
// 获取当前文件的目录路径
|
| 10 |
-
const __filename = fileURLToPath(import.meta.url);
|
| 11 |
-
const __dirname = dirname(__filename);
|
| 12 |
-
|
| 13 |
-
// 加载环境变量
|
| 14 |
-
dotenv.config({ path: join(dirname(__dirname), '.env') });
|
| 15 |
-
|
| 16 |
-
// 日志配置
|
| 17 |
-
const logger = {
|
| 18 |
-
info: (message) => console.log(chalk.blue(`[ProxyServer] ${message}`)),
|
| 19 |
-
error: (message) => console.error(chalk.red(`[ProxyServer] ${message}`)),
|
| 20 |
-
warning: (message) => console.warn(chalk.yellow(`[ProxyServer] ${message}`)),
|
| 21 |
-
success: (message) => console.log(chalk.green(`[ProxyServer] ${message}`)),
|
| 22 |
-
};
|
| 23 |
-
|
| 24 |
-
class ProxyServer {
|
| 25 |
-
constructor() {
|
| 26 |
-
this.proxyProcess = null;
|
| 27 |
-
this.platform = process.env.PROXY_SERVER_PLATFORM || 'auto';
|
| 28 |
-
this.port = process.env.PROXY_SERVER_PORT || 10655;
|
| 29 |
-
this.
|
| 30 |
-
this.
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
return '
|
| 49 |
-
}
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
} else {
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
case '
|
| 67 |
-
return join(proxyDir, '
|
| 68 |
-
case '
|
| 69 |
-
return join(proxyDir, '
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
})
|
| 103 |
-
|
| 104 |
-
//
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
logger.error(
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
//
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
}
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
}
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
export { proxyServer };
|
|
|
|
| 1 |
+
import { spawn } from 'child_process';
|
| 2 |
+
import { fileURLToPath } from 'url';
|
| 3 |
+
import { dirname, join } from 'path';
|
| 4 |
+
import fs from 'fs';
|
| 5 |
+
import os from 'os';
|
| 6 |
+
import dotenv from 'dotenv';
|
| 7 |
+
import chalk from 'chalk';
|
| 8 |
+
|
| 9 |
+
// 获取当前文件的目录路径
|
| 10 |
+
const __filename = fileURLToPath(import.meta.url);
|
| 11 |
+
const __dirname = dirname(__filename);
|
| 12 |
+
|
| 13 |
+
// 加载环境变量
|
| 14 |
+
dotenv.config({ path: join(dirname(__dirname), '.env') });
|
| 15 |
+
|
| 16 |
+
// 日志配置
|
| 17 |
+
const logger = {
|
| 18 |
+
info: (message) => console.log(chalk.blue(`[ProxyServer] ${message}`)),
|
| 19 |
+
error: (message) => console.error(chalk.red(`[ProxyServer] ${message}`)),
|
| 20 |
+
warning: (message) => console.warn(chalk.yellow(`[ProxyServer] ${message}`)),
|
| 21 |
+
success: (message) => console.log(chalk.green(`[ProxyServer] ${message}`)),
|
| 22 |
+
};
|
| 23 |
+
|
| 24 |
+
class ProxyServer {
|
| 25 |
+
constructor() {
|
| 26 |
+
this.proxyProcess = null;
|
| 27 |
+
this.platform = process.env.PROXY_SERVER_PLATFORM || 'auto';
|
| 28 |
+
this.port = process.env.PROXY_SERVER_PORT || 10655;
|
| 29 |
+
this.logPath = process.env.PROXY_SERVER_LOG_PATH || '/tmp/proxy_server.log';
|
| 30 |
+
this.enabled = process.env.ENABLE_PROXY_SERVER === 'true';
|
| 31 |
+
this.proxyAuthToken = process.env.PROXY_AUTH_TOKEN || 'default_token';
|
| 32 |
+
this.logStream = null;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
// 获取当前系统平台
|
| 36 |
+
detectPlatform() {
|
| 37 |
+
if (this.platform !== 'auto') {
|
| 38 |
+
return this.platform;
|
| 39 |
+
}
|
| 40 |
+
|
| 41 |
+
const platform = os.platform();
|
| 42 |
+
const arch = os.arch();
|
| 43 |
+
|
| 44 |
+
if (platform === 'win32') {
|
| 45 |
+
return 'windows';
|
| 46 |
+
} else if (platform === 'linux') {
|
| 47 |
+
if (arch === 'arm64') {
|
| 48 |
+
return 'android';
|
| 49 |
+
} else {
|
| 50 |
+
return 'linux';
|
| 51 |
+
}
|
| 52 |
+
} else if (platform === 'android') {
|
| 53 |
+
return 'android';
|
| 54 |
+
} else {
|
| 55 |
+
logger.warning(`未知平台: ${platform}, ${arch}, 默认使用linux版本`);
|
| 56 |
+
return 'linux';
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
// 获取代理服务器可执行文件路径
|
| 61 |
+
getProxyServerPath() {
|
| 62 |
+
const platform = this.detectPlatform();
|
| 63 |
+
const proxyDir = join(__dirname, 'proxy');
|
| 64 |
+
|
| 65 |
+
switch (platform) {
|
| 66 |
+
case 'windows':
|
| 67 |
+
return join(proxyDir, 'chrome_proxy_server_windows_amd64.exe');
|
| 68 |
+
case 'linux':
|
| 69 |
+
return join(proxyDir, 'chrome_proxy_server_linux_amd64');
|
| 70 |
+
case 'android':
|
| 71 |
+
return join(proxyDir, 'chrome_proxy_server_android_arm64');
|
| 72 |
+
default:
|
| 73 |
+
logger.error(`不支持的平台: ${platform}`);
|
| 74 |
+
return null;
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
// 启动代理服务器
|
| 79 |
+
async start() {
|
| 80 |
+
if (!this.enabled) {
|
| 81 |
+
logger.info('代理服务器未启用,跳过启动');
|
| 82 |
+
return;
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
if (this.proxyProcess) {
|
| 86 |
+
logger.warning('代理服务器已经在运行中');
|
| 87 |
+
return;
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
const proxyServerPath = this.getProxyServerPath();
|
| 91 |
+
if (!proxyServerPath) {
|
| 92 |
+
logger.error('无法获取代理服务器路径');
|
| 93 |
+
return;
|
| 94 |
+
}
|
| 95 |
+
|
| 96 |
+
try {
|
| 97 |
+
// 确保可执行文件有执行权限(在Linux/Android上)
|
| 98 |
+
// 在Hugging Face环境中,文件权限应在Dockerfile中设置,运行时修改可能会失败
|
| 99 |
+
// if (this.detectPlatform() !== 'windows') {
|
| 100 |
+
// try {
|
| 101 |
+
// fs.chmodSync(proxyServerPath, 0o755);
|
| 102 |
+
// } catch (err) {
|
| 103 |
+
// logger.warning(`无法设置执行权限: ${err.message}`);
|
| 104 |
+
// }
|
| 105 |
+
// }
|
| 106 |
+
|
| 107 |
+
// 创建日志文件
|
| 108 |
+
this.logStream = fs.createWriteStream(this.logPath, { flags: 'a' });
|
| 109 |
+
|
| 110 |
+
// 修复 stdio 参数问题
|
| 111 |
+
// 启动代理服务器进程
|
| 112 |
+
this.proxyProcess = spawn(proxyServerPath, [
|
| 113 |
+
'--port', this.port.toString(),
|
| 114 |
+
'--token', this.proxyAuthToken
|
| 115 |
+
], {
|
| 116 |
+
stdio: ['ignore', 'pipe', 'pipe'], // 使用pipe而不是直接传递流
|
| 117 |
+
detached: false
|
| 118 |
+
});
|
| 119 |
+
|
| 120 |
+
// 将进程的输出重定向到日志文件
|
| 121 |
+
if (this.proxyProcess.stdout) {
|
| 122 |
+
this.proxyProcess.stdout.pipe(this.logStream);
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
if (this.proxyProcess.stderr) {
|
| 126 |
+
this.proxyProcess.stderr.pipe(this.logStream);
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
// 设置进程事件处理
|
| 130 |
+
this.proxyProcess.on('error', (err) => {
|
| 131 |
+
logger.error(`代理服务器启动失败: ${err.message}`);
|
| 132 |
+
this.proxyProcess = null;
|
| 133 |
+
if (this.logStream) {
|
| 134 |
+
this.logStream.end();
|
| 135 |
+
this.logStream = null;
|
| 136 |
+
}
|
| 137 |
+
});
|
| 138 |
+
|
| 139 |
+
this.proxyProcess.on('exit', (code, signal) => {
|
| 140 |
+
logger.info(`代理服务器已退出,退出码: ${code}, 信号: ${signal}`);
|
| 141 |
+
this.proxyProcess = null;
|
| 142 |
+
if (this.logStream) {
|
| 143 |
+
this.logStream.end();
|
| 144 |
+
this.logStream = null;
|
| 145 |
+
}
|
| 146 |
+
});
|
| 147 |
+
|
| 148 |
+
// 等待一段时间,确保服务器启动
|
| 149 |
+
await new Promise(resolve => setTimeout(resolve, 1000));
|
| 150 |
+
|
| 151 |
+
if (this.proxyProcess && this.proxyProcess.exitCode === null) {
|
| 152 |
+
logger.success(`代理服务器已启动,端口: ${this.port}, 日志文件: ${this.logPath}`);
|
| 153 |
+
return true;
|
| 154 |
+
} else {
|
| 155 |
+
logger.error('代理服务器���动失败');
|
| 156 |
+
if (this.logStream) {
|
| 157 |
+
this.logStream.end();
|
| 158 |
+
this.logStream = null;
|
| 159 |
+
}
|
| 160 |
+
return false;
|
| 161 |
+
}
|
| 162 |
+
} catch (error) {
|
| 163 |
+
logger.error(`启动代理服务器时出错: ${error.message}`);
|
| 164 |
+
if (this.logStream) {
|
| 165 |
+
this.logStream.end();
|
| 166 |
+
this.logStream = null;
|
| 167 |
+
}
|
| 168 |
+
return false;
|
| 169 |
+
}
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
// 停止代理服务器
|
| 173 |
+
stop() {
|
| 174 |
+
if (!this.proxyProcess) {
|
| 175 |
+
//logger.info('代理服务器已关闭');
|
| 176 |
+
return;
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
try {
|
| 180 |
+
// 在Windows上使用taskkill确保子进程也被终止
|
| 181 |
+
if (this.detectPlatform() === 'windows' && this.proxyProcess.pid) {
|
| 182 |
+
spawn('taskkill', ['/pid', this.proxyProcess.pid, '/f', '/t']);
|
| 183 |
+
} else {
|
| 184 |
+
// 在Linux/Android上使用kill信号
|
| 185 |
+
this.proxyProcess.kill('SIGTERM');
|
| 186 |
+
}
|
| 187 |
+
|
| 188 |
+
logger.success('代理服务器已停止');
|
| 189 |
+
} catch (error) {
|
| 190 |
+
logger.error(`停止代理服务器时出错: ${error.message}`);
|
| 191 |
+
} finally {
|
| 192 |
+
this.proxyProcess = null;
|
| 193 |
+
if (this.logStream) {
|
| 194 |
+
this.logStream.end();
|
| 195 |
+
this.logStream = null;
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
}
|
| 200 |
+
|
| 201 |
+
// 创建单例
|
| 202 |
+
const proxyServer = new ProxyServer();
|
| 203 |
+
|
| 204 |
+
// 导出
|
| 205 |
export { proxyServer };
|
src/lightweight-client-express.js
CHANGED
|
@@ -321,7 +321,7 @@ app.get('/cookies/status', authenticate, (req, res) => {
|
|
| 321 |
const PORT = process.env.PORT || 7860;
|
| 322 |
|
| 323 |
// 设置代理池日志级别为warn,减少详细日志输出
|
| 324 |
-
proxyPool.logLevel = '
|
| 325 |
|
| 326 |
// 初始化并启动服务器
|
| 327 |
initialize().then(() => {
|
|
|
|
| 321 |
const PORT = process.env.PORT || 7860;
|
| 322 |
|
| 323 |
// 设置代理池日志级别为warn,减少详细日志输出
|
| 324 |
+
proxyPool.logLevel = 'error';
|
| 325 |
|
| 326 |
// 初始化并启动服务器
|
| 327 |
initialize().then(() => {
|
src/lightweight-client.js
CHANGED
|
@@ -14,8 +14,6 @@ import {
|
|
| 14 |
import { proxyPool } from './ProxyPool.js';
|
| 15 |
import { proxyServer } from './ProxyServer.js';
|
| 16 |
import { cookieManager } from './CookieManager.js';
|
| 17 |
-
import { HttpsProxyAgent } from 'https-proxy-agent';
|
| 18 |
-
import dns from 'dns';
|
| 19 |
|
| 20 |
// 获取当前文件的目录路径
|
| 21 |
const __filename = fileURLToPath(import.meta.url);
|
|
@@ -382,50 +380,34 @@ async function fetchNotionResponse(chunkQueue, notionRequestBody, headers, notio
|
|
| 382 |
// 创建fetch选项
|
| 383 |
const fetchOptions = {
|
| 384 |
method: 'POST',
|
| 385 |
-
headers:
|
|
|
|
|
|
|
|
|
|
|
|
|
| 386 |
body: JSON.stringify(notionRequestBody),
|
| 387 |
-
timeout: 120000 // 120s
|
| 388 |
};
|
| 389 |
|
| 390 |
-
//
|
| 391 |
if (USE_NATIVE_PROXY_POOL && ENABLE_PROXY_SERVER && !PROXY_URL) {
|
| 392 |
-
|
| 393 |
-
if (proxy)
|
| 394 |
-
|
| 395 |
-
logger.info(
|
| 396 |
-
} else {
|
| 397 |
-
logger.warning("原生代理池已启用但没有可用的代理,将不使用代理");
|
| 398 |
}
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
if (proxy) {
|
| 402 |
-
fetchOptions.agent = new HttpsProxyAgent(proxy.url);
|
| 403 |
-
logger.info(`使用原生代理池: ${proxy.url} (来自 ${proxy.source})`);
|
| 404 |
-
} else {
|
| 405 |
-
logger.warning("原生代理池已启用但没有可用的代理,将不使用代理");
|
| 406 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 407 |
}else if(PROXY_URL){
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
dns.lookup(hostname, { family: 6 }, (err, address, family) => {
|
| 411 |
-
if (err) {
|
| 412 |
-
// 如果 IPv6 查询失败,则回退到 IPv4
|
| 413 |
-
dns.lookup(hostname, { family: 4 }, (err, address, family) => {
|
| 414 |
-
callback(err, address, family);
|
| 415 |
-
});
|
| 416 |
-
} else {
|
| 417 |
-
callback(err, address, family);
|
| 418 |
-
}
|
| 419 |
-
});
|
| 420 |
-
};
|
| 421 |
-
fetchOptions.agent = new HttpsProxyAgent(PROXY_URL, { lookup: customLookup });
|
| 422 |
logger.info(`使用代理: ${PROXY_URL}`);
|
| 423 |
}
|
| 424 |
-
else if (ENABLE_PROXY_SERVER && !USE_NATIVE_PROXY_POOL && !PROXY_URL){
|
| 425 |
-
const proxy = await proxyPool.getProxy();
|
| 426 |
-
fetchOptions.agent = new HttpsProxyAgent(proxy.url);
|
| 427 |
-
logger.info(`使用代理: ${proxy.url} (来自 ${proxy.source})`);
|
| 428 |
-
}
|
| 429 |
let response = null;
|
| 430 |
// 发送请求
|
| 431 |
if (ENABLE_PROXY_SERVER && USE_NATIVE_PROXY_POOL){
|
|
@@ -873,7 +855,7 @@ async function initialize() {
|
|
| 873 |
if (process.env.USE_NATIVE_PROXY_POOL === 'true') {
|
| 874 |
logger.info(`正在初始化本地代理池...`);
|
| 875 |
// 设置代理池的日志级别为warn,减少详细日志输出
|
| 876 |
-
proxyPool.logLevel = '
|
| 877 |
// 启用进度条显示
|
| 878 |
proxyPool.showProgressBar = true;
|
| 879 |
|
|
|
|
| 14 |
import { proxyPool } from './ProxyPool.js';
|
| 15 |
import { proxyServer } from './ProxyServer.js';
|
| 16 |
import { cookieManager } from './CookieManager.js';
|
|
|
|
|
|
|
| 17 |
|
| 18 |
// 获取当前文件的目录路径
|
| 19 |
const __filename = fileURLToPath(import.meta.url);
|
|
|
|
| 380 |
// 创建fetch选项
|
| 381 |
const fetchOptions = {
|
| 382 |
method: 'POST',
|
| 383 |
+
headers: {
|
| 384 |
+
...headers,
|
| 385 |
+
'user-agent': window.navigator.userAgent,
|
| 386 |
+
'Cookie': notionCookie
|
| 387 |
+
},
|
| 388 |
body: JSON.stringify(notionRequestBody),
|
|
|
|
| 389 |
};
|
| 390 |
|
| 391 |
+
// 添加代理配置(如果有)
|
| 392 |
if (USE_NATIVE_PROXY_POOL && ENABLE_PROXY_SERVER && !PROXY_URL) {
|
| 393 |
+
proxy = proxyPool.getProxy();
|
| 394 |
+
if (proxy !== null)
|
| 395 |
+
{
|
| 396 |
+
logger.info(`使用代理: ${proxy.full}`);
|
|
|
|
|
|
|
| 397 |
}
|
| 398 |
+
else{
|
| 399 |
+
logger.warning(`没有可用代理`);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 400 |
}
|
| 401 |
+
} else if(USE_NATIVE_PROXY_POOL&&!PROXY_URL&&!ENABLE_PROXY_SERVER) {
|
| 402 |
+
const { HttpsProxyAgent } = await import('https-proxy-agent');
|
| 403 |
+
proxy = proxyPool.getProxy();
|
| 404 |
+
fetchOptions.agent = new HttpsProxyAgent(proxy.full);
|
| 405 |
+
logger.info(`使用代理: ${proxy.full}`);
|
| 406 |
}else if(PROXY_URL){
|
| 407 |
+
const { HttpsProxyAgent } = await import('https-proxy-agent');
|
| 408 |
+
fetchOptions.agent = new HttpsProxyAgent(PROXY_URL);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 409 |
logger.info(`使用代理: ${PROXY_URL}`);
|
| 410 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 411 |
let response = null;
|
| 412 |
// 发送请求
|
| 413 |
if (ENABLE_PROXY_SERVER && USE_NATIVE_PROXY_POOL){
|
|
|
|
| 855 |
if (process.env.USE_NATIVE_PROXY_POOL === 'true') {
|
| 856 |
logger.info(`正在初始化本地代理池...`);
|
| 857 |
// 设置代理池的日志级别为warn,减少详细日志输出
|
| 858 |
+
proxyPool.logLevel = 'error';
|
| 859 |
// 启用进度条显示
|
| 860 |
proxyPool.showProgressBar = true;
|
| 861 |
|
src/proxy/chrome_proxy_server_android_arm64
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e4fce54fee0fe3acc4d255517134dcabb9dc3086e9dbe571bc9f8f216ba64039
|
| 3 |
+
size 12863727
|
src/proxy/chrome_proxy_server_linux_amd64
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2e33e31ff51fb711daff95ca41f84e5f56dbfc9c57b90df3761bc1d69e57bfa9
|
| 3 |
+
size 12845606
|
src/proxy/chrome_proxy_server_windows_amd64.exe
CHANGED
|
@@ -1,3 +1,3 @@
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:
|
| 3 |
-
size
|
|
|
|
| 1 |
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:bb09f4ed7261e2f0db9be7b1c37df628549df6243aee4c6e3b25d3e9452028f6
|
| 3 |
+
size 12961280
|