597 lines
16 KiB
Go
597 lines
16 KiB
Go
package client
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
|
|
"codex-pool/internal/proxyutil"
|
|
|
|
"github.com/Noooste/azuretls-client"
|
|
fhttp "github.com/Noooste/fhttp"
|
|
"github.com/andybalholm/brotli"
|
|
http2 "github.com/bogdanfinn/fhttp"
|
|
tls_client "github.com/bogdanfinn/tls-client"
|
|
)
|
|
|
|
// ClientType 客户端类型
|
|
type ClientType int
|
|
|
|
const (
|
|
ClientTypeTLS ClientType = iota // bogdanfinn/tls-client
|
|
ClientTypeAzureTLS // Noooste/azuretls-client
|
|
)
|
|
|
|
// TLSClient 统一的 HTTP 客户端接口，支持多种 TLS 指纹和浏览器模拟
|
|
type TLSClient struct {
|
|
// 内部客户端
|
|
tlsClient tls_client.HttpClient
|
|
azureSession *azuretls.Session
|
|
clientType ClientType
|
|
|
|
// 指纹信息
|
|
fingerprint BrowserFingerprint
|
|
userAgent string
|
|
acceptLang string
|
|
proxy string
|
|
}
|
|
|
|
// 语言偏好池
|
|
var languagePrefs = []string{
|
|
"en-US,en;q=0.9",
|
|
"en-GB,en;q=0.9,en-US;q=0.8",
|
|
"en-US,en;q=0.9,de;q=0.8",
|
|
"de-DE,de;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
"en-US,en;q=0.9,fr;q=0.8",
|
|
"en-US,en;q=0.9,es;q=0.8",
|
|
"fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
"es-ES,es;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
}
|
|
|
|
// New 创建一个新的 TLS 客户端 (使用随机指纹)
|
|
func New(proxyStr string) (*TLSClient, error) {
|
|
// 获取随机桌面端指纹
|
|
fp := GetRandomDesktopFingerprint()
|
|
return NewWithFingerprint(fp, proxyStr)
|
|
}
|
|
|
|
// NewWithFingerprint 使用指定指纹创建客户端
|
|
func NewWithFingerprint(fp BrowserFingerprint, proxyStr string) (client *TLSClient, err error) {
|
|
// 添加 panic 恢复
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
err = fmt.Errorf("创建客户端时发生 panic: %v", r)
|
|
client = nil
|
|
}
|
|
}()
|
|
|
|
acceptLang := languagePrefs[rand.Intn(len(languagePrefs))]
|
|
userAgent := generateUserAgent(fp)
|
|
|
|
client = &TLSClient{
|
|
fingerprint: fp,
|
|
userAgent: userAgent,
|
|
acceptLang: acceptLang,
|
|
proxy: proxyStr,
|
|
}
|
|
|
|
// 根据指纹类型创建对应的客户端
|
|
switch fp.Type {
|
|
case FingerprintAzureTLS:
|
|
return createAzureTLSClient(client, fp, proxyStr)
|
|
default:
|
|
return createTLSClient(client, fp, proxyStr)
|
|
}
|
|
}
|
|
|
|
// createTLSClient 创建 tls-client 客户端
|
|
func createTLSClient(c *TLSClient, fp BrowserFingerprint, proxyStr string) (*TLSClient, error) {
|
|
jar := tls_client.NewCookieJar()
|
|
|
|
options := []tls_client.HttpClientOption{
|
|
tls_client.WithTimeoutSeconds(90),
|
|
tls_client.WithClientProfile(fp.TLSProfile),
|
|
tls_client.WithRandomTLSExtensionOrder(),
|
|
tls_client.WithCookieJar(jar),
|
|
tls_client.WithInsecureSkipVerify(),
|
|
tls_client.WithNotFollowRedirects(),
|
|
}
|
|
|
|
if proxyStr != "" {
|
|
normalized, err := proxyutil.Normalize(proxyStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
options = append(options, tls_client.WithProxyUrl(normalized))
|
|
}
|
|
|
|
tlsClient, err := tls_client.NewHttpClient(tls_client.NewNoopLogger(), options...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
c.tlsClient = tlsClient
|
|
c.clientType = ClientTypeTLS
|
|
return c, nil
|
|
}
|
|
|
|
// createAzureTLSClient 创建 azuretls-client 客户端
|
|
func createAzureTLSClient(c *TLSClient, fp BrowserFingerprint, proxyStr string) (*TLSClient, error) {
|
|
session := azuretls.NewSession()
|
|
|
|
browser := fp.AzureBrowser
|
|
if browser == "" {
|
|
browser = azuretls.Chrome
|
|
}
|
|
|
|
session.Browser = browser
|
|
session.GetClientHelloSpec = azuretls.GetBrowserClientHelloFunc(browser)
|
|
session.SetTimeout(90 * time.Second)
|
|
|
|
if proxyStr != "" {
|
|
normalized, err := proxyutil.Normalize(proxyStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := session.SetProxy(normalized); err != nil {
|
|
return nil, fmt.Errorf("azuretls set proxy failed: %w", err)
|
|
}
|
|
}
|
|
|
|
session.MaxRedirects = 0
|
|
session.InsecureSkipVerify = true
|
|
|
|
c.azureSession = session
|
|
c.clientType = ClientTypeAzureTLS
|
|
return c, nil
|
|
}
|
|
|
|
// generateUserAgent 根据指纹生成 User-Agent
|
|
func generateUserAgent(fp BrowserFingerprint) string {
|
|
winVersions := []string{"Windows NT 10.0; Win64; x64", "Windows NT 11.0; Win64; x64"}
|
|
macVersions := []string{"10_15_7", "11_0_0", "12_0_0", "13_0_0", "14_0", "14_5", "15_0", "15_2"}
|
|
linuxVersions := []string{"X11; Linux x86_64", "X11; Ubuntu; Linux x86_64", "X11; Fedora; Linux x86_64"}
|
|
|
|
version := fp.Version
|
|
if version == "latest" {
|
|
switch fp.Browser {
|
|
case "chrome":
|
|
version = "133"
|
|
case "firefox":
|
|
version = "135"
|
|
case "safari":
|
|
version = "18.3"
|
|
case "edge":
|
|
version = "133"
|
|
case "opera":
|
|
version = "117"
|
|
default:
|
|
version = "133"
|
|
}
|
|
}
|
|
|
|
switch fp.Browser {
|
|
case "chrome":
|
|
switch fp.Platform {
|
|
case "Windows":
|
|
return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
|
|
winVersions[rand.Intn(len(winVersions))], version)
|
|
case "macOS":
|
|
return fmt.Sprintf("Mozilla/5.0 (Macintosh; Intel Mac OS X %s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
|
|
macVersions[rand.Intn(len(macVersions))], version)
|
|
case "Linux":
|
|
return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36",
|
|
linuxVersions[rand.Intn(len(linuxVersions))], version)
|
|
case "Android":
|
|
return fmt.Sprintf("Mozilla/5.0 (Linux; Android 14; SM-S918B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Mobile Safari/537.36", version)
|
|
}
|
|
|
|
case "firefox":
|
|
switch fp.Platform {
|
|
case "Windows":
|
|
return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s.0) Gecko/20100101 Firefox/%s.0",
|
|
winVersions[rand.Intn(len(winVersions))], version, version)
|
|
case "macOS":
|
|
return fmt.Sprintf("Mozilla/5.0 (Macintosh; Intel Mac OS X %s; rv:%s.0) Gecko/20100101 Firefox/%s.0",
|
|
macVersions[rand.Intn(len(macVersions))], version, version)
|
|
case "Linux":
|
|
return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s.0) Gecko/20100101 Firefox/%s.0",
|
|
linuxVersions[rand.Intn(len(linuxVersions))], version, version)
|
|
}
|
|
|
|
case "safari":
|
|
if fp.Mobile {
|
|
iosVersions := map[string]string{"18.5": "18_5", "18.0": "18_0", "17.0": "17_0", "16.0": "16_0", "15.6": "15_6", "15.5": "15_5"}
|
|
iosVer := iosVersions[version]
|
|
if iosVer == "" {
|
|
iosVer = "18_0"
|
|
}
|
|
if fp.Platform == "iPadOS" {
|
|
return fmt.Sprintf("Mozilla/5.0 (iPad; CPU OS %s like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/%s Mobile/15E148 Safari/604.1", iosVer, version)
|
|
}
|
|
return fmt.Sprintf("Mozilla/5.0 (iPhone; CPU iPhone OS %s like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/%s Mobile/15E148 Safari/604.1", iosVer, version)
|
|
}
|
|
return fmt.Sprintf("Mozilla/5.0 (Macintosh; Intel Mac OS X %s) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/%s Safari/605.1.15",
|
|
macVersions[rand.Intn(len(macVersions))], version)
|
|
|
|
case "opera":
|
|
chromeVer := map[string]string{"91": "118", "90": "117", "89": "116"}[version]
|
|
if chromeVer == "" {
|
|
chromeVer = "118"
|
|
}
|
|
return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36 OPR/%s.0.0.0",
|
|
winVersions[rand.Intn(len(winVersions))], chromeVer, version)
|
|
|
|
case "okhttp":
|
|
return "okhttp/4.12.0"
|
|
}
|
|
|
|
return fmt.Sprintf("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s.0.0.0 Safari/537.36", version)
|
|
}
|
|
|
|
// getDefaultHeaders 获取默认请求头
|
|
func (c *TLSClient) getDefaultHeaders() map[string]string {
|
|
headers := map[string]string{
|
|
"User-Agent": c.userAgent,
|
|
"Accept-Language": c.acceptLang,
|
|
"Accept-Encoding": "gzip, deflate, br",
|
|
}
|
|
|
|
fp := c.fingerprint
|
|
switch fp.Browser {
|
|
case "firefox":
|
|
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
|
|
headers["Upgrade-Insecure-Requests"] = "1"
|
|
headers["Sec-Fetch-Dest"] = "document"
|
|
headers["Sec-Fetch-Mode"] = "navigate"
|
|
headers["Sec-Fetch-Site"] = "none"
|
|
headers["Sec-Fetch-User"] = "?1"
|
|
headers["DNT"] = "1"
|
|
|
|
case "safari":
|
|
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
|
|
|
case "okhttp":
|
|
headers["Accept"] = "*/*"
|
|
headers["Accept-Encoding"] = "gzip"
|
|
|
|
default: // chrome, opera
|
|
secChUa := c.generateSecChUa()
|
|
headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"
|
|
headers["Cache-Control"] = "max-age=0"
|
|
headers["Sec-Ch-Ua"] = secChUa
|
|
headers["Sec-Ch-Ua-Mobile"] = "?0"
|
|
if fp.Mobile {
|
|
headers["Sec-Ch-Ua-Mobile"] = "?1"
|
|
}
|
|
headers["Sec-Ch-Ua-Platform"] = c.getPlatformHeader()
|
|
headers["Sec-Fetch-Dest"] = "document"
|
|
headers["Sec-Fetch-Mode"] = "navigate"
|
|
headers["Sec-Fetch-Site"] = "none"
|
|
headers["Sec-Fetch-User"] = "?1"
|
|
headers["Upgrade-Insecure-Requests"] = "1"
|
|
}
|
|
|
|
return headers
|
|
}
|
|
|
|
// generateSecChUa 生成 Sec-Ch-Ua 头
|
|
func (c *TLSClient) generateSecChUa() string {
|
|
ver := c.fingerprint.Version
|
|
switch c.fingerprint.Browser {
|
|
case "opera":
|
|
return fmt.Sprintf(`"Opera";v="%s", "Chromium";v="118", "Not(A:Brand";v="99"`, ver)
|
|
default:
|
|
notABrands := []string{`"Not(A:Brand";v="99"`, `"Not A(Brand";v="99"`, `"Not/A)Brand";v="99"`}
|
|
return fmt.Sprintf(`"Chromium";v="%s", %s, "Google Chrome";v="%s"`, ver, notABrands[rand.Intn(len(notABrands))], ver)
|
|
}
|
|
}
|
|
|
|
// getPlatformHeader 获取平台头
|
|
func (c *TLSClient) getPlatformHeader() string {
|
|
switch c.fingerprint.Platform {
|
|
case "macOS":
|
|
return `"macOS"`
|
|
case "Linux":
|
|
return `"Linux"`
|
|
case "iOS", "iPadOS":
|
|
return `"iOS"`
|
|
case "Android":
|
|
return `"Android"`
|
|
default:
|
|
return `"Windows"`
|
|
}
|
|
}
|
|
|
|
// Do 执行 HTTP 请求
|
|
func (c *TLSClient) Do(req *http.Request) (*http.Response, error) {
|
|
switch c.clientType {
|
|
case ClientTypeAzureTLS:
|
|
return c.doAzureTLS(req)
|
|
default:
|
|
return c.doTLS(req)
|
|
}
|
|
}
|
|
|
|
// doTLS 使用 tls-client 执行请求
|
|
func (c *TLSClient) doTLS(req *http.Request) (*http.Response, error) {
|
|
fhttpReq, err := http2.NewRequest(req.Method, req.URL.String(), req.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
for key, value := range c.getDefaultHeaders() {
|
|
if req.Header.Get(key) == "" {
|
|
fhttpReq.Header.Set(key, value)
|
|
}
|
|
}
|
|
|
|
for key, values := range req.Header {
|
|
for _, value := range values {
|
|
fhttpReq.Header.Set(key, value)
|
|
}
|
|
}
|
|
|
|
resp, err := c.tlsClient.Do(fhttpReq)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
finalReq := req
|
|
if resp.Request != nil && resp.Request.URL != nil {
|
|
finalReq = &http.Request{
|
|
Method: resp.Request.Method,
|
|
URL: (*url.URL)(resp.Request.URL),
|
|
Header: http.Header(resp.Request.Header),
|
|
}
|
|
}
|
|
|
|
return &http.Response{
|
|
Status: resp.Status,
|
|
StatusCode: resp.StatusCode,
|
|
Proto: resp.Proto,
|
|
ProtoMajor: resp.ProtoMajor,
|
|
ProtoMinor: resp.ProtoMinor,
|
|
Header: http.Header(resp.Header),
|
|
Body: resp.Body,
|
|
ContentLength: resp.ContentLength,
|
|
TransferEncoding: resp.TransferEncoding,
|
|
Close: resp.Close,
|
|
Uncompressed: resp.Uncompressed,
|
|
Request: finalReq,
|
|
}, nil
|
|
}
|
|
|
|
// doAzureTLS 使用 azuretls 执行请求
|
|
func (c *TLSClient) doAzureTLS(req *http.Request) (*http.Response, error) {
|
|
orderedHeaders := azuretls.OrderedHeaders{}
|
|
for key, value := range c.getDefaultHeaders() {
|
|
if req.Header.Get(key) == "" {
|
|
orderedHeaders = append(orderedHeaders, []string{key, value})
|
|
}
|
|
}
|
|
for key, values := range req.Header {
|
|
if len(values) > 0 {
|
|
orderedHeaders = append(orderedHeaders, []string{key, values[0]})
|
|
}
|
|
}
|
|
|
|
azureReq := &azuretls.Request{
|
|
Method: req.Method,
|
|
Url: req.URL.String(),
|
|
OrderedHeaders: orderedHeaders,
|
|
DisableRedirects: true,
|
|
}
|
|
|
|
if req.Body != nil {
|
|
bodyBytes, _ := io.ReadAll(req.Body)
|
|
azureReq.Body = bodyBytes
|
|
}
|
|
|
|
resp, err := c.azureSession.Do(azureReq)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
respHeaders := make(http.Header)
|
|
for key, values := range resp.Header {
|
|
respHeaders[key] = values
|
|
}
|
|
|
|
return &http.Response{
|
|
Status: fmt.Sprintf("%d %s", resp.StatusCode, http.StatusText(resp.StatusCode)),
|
|
StatusCode: resp.StatusCode,
|
|
Header: respHeaders,
|
|
Body: io.NopCloser(bytes.NewReader(resp.Body)),
|
|
Request: req,
|
|
}, nil
|
|
}
|
|
|
|
// DoWithRedirect 执行请求并手动处理重定向
|
|
func (c *TLSClient) DoWithRedirect(req *http.Request, maxRedirects int) (*http.Response, error) {
|
|
for i := 0; i <= maxRedirects; i++ {
|
|
resp, err := c.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if resp.StatusCode >= 300 && resp.StatusCode < 400 {
|
|
location := resp.Header.Get("Location")
|
|
if location == "" {
|
|
return resp, nil
|
|
}
|
|
|
|
redirectURL, err := url.Parse(location)
|
|
if err != nil {
|
|
return resp, nil
|
|
}
|
|
|
|
if !redirectURL.IsAbs() {
|
|
redirectURL = req.URL.ResolveReference(redirectURL)
|
|
}
|
|
|
|
req, err = http.NewRequest(http.MethodGet, redirectURL.String(), nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp.Body.Close()
|
|
continue
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
return nil, fmt.Errorf("too many redirects")
|
|
}
|
|
|
|
// Get 执行 GET 请求
|
|
func (c *TLSClient) Get(urlStr string) (*http.Response, error) {
|
|
req, err := http.NewRequest(http.MethodGet, urlStr, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return c.DoWithRedirect(req, 10)
|
|
}
|
|
|
|
// Post 执行 POST 请求
|
|
func (c *TLSClient) Post(urlStr string, contentType string, body io.Reader) (*http.Response, error) {
|
|
req, err := http.NewRequest(http.MethodPost, urlStr, body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", contentType)
|
|
return c.Do(req)
|
|
}
|
|
|
|
// PostJSON 执行 POST JSON 请求
|
|
func (c *TLSClient) PostJSON(urlStr string, body io.Reader) (*http.Response, error) {
|
|
return c.Post(urlStr, "application/json", body)
|
|
}
|
|
|
|
// GetCookie 获取指定 URL 的 Cookie
|
|
func (c *TLSClient) GetCookie(urlStr string, name string) string {
|
|
u, err := url.Parse(urlStr)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
|
|
// azuretls 使用标准的 http.CookieJar
|
|
if c.clientType == ClientTypeAzureTLS {
|
|
if c.azureSession == nil || c.azureSession.CookieJar == nil {
|
|
return ""
|
|
}
|
|
cookies := c.azureSession.CookieJar.Cookies(u)
|
|
for _, cookie := range cookies {
|
|
if cookie.Name == name {
|
|
return cookie.Value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// tls-client
|
|
if c.tlsClient == nil {
|
|
return ""
|
|
}
|
|
cookies := c.tlsClient.GetCookies(u)
|
|
for _, cookie := range cookies {
|
|
if cookie.Name == name {
|
|
return cookie.Value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// SetCookie 设置 Cookie
|
|
func (c *TLSClient) SetCookie(urlStr string, cookie *http.Cookie) {
|
|
u, err := url.Parse(urlStr)
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
// azuretls 使用 Noooste/fhttp 的 Cookie 类型
|
|
if c.clientType == ClientTypeAzureTLS {
|
|
if c.azureSession == nil || c.azureSession.CookieJar == nil {
|
|
return
|
|
}
|
|
c.azureSession.CookieJar.SetCookies(u, []*fhttp.Cookie{
|
|
{
|
|
Name: cookie.Name,
|
|
Value: cookie.Value,
|
|
Path: cookie.Path,
|
|
Domain: cookie.Domain,
|
|
},
|
|
})
|
|
return
|
|
}
|
|
|
|
// tls-client
|
|
if c.tlsClient == nil {
|
|
return
|
|
}
|
|
c.tlsClient.SetCookies(u, []*http2.Cookie{
|
|
{
|
|
Name: cookie.Name,
|
|
Value: cookie.Value,
|
|
Path: cookie.Path,
|
|
Domain: cookie.Domain,
|
|
},
|
|
})
|
|
}
|
|
|
|
// ReadBody 读取响应体并自动处理压缩
|
|
func ReadBody(resp *http.Response) ([]byte, error) {
|
|
defer resp.Body.Close()
|
|
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
switch resp.Header.Get("Content-Encoding") {
|
|
case "gzip":
|
|
gzReader, err := gzip.NewReader(bytes.NewReader(data))
|
|
if err != nil {
|
|
return data, nil
|
|
}
|
|
defer gzReader.Close()
|
|
return io.ReadAll(gzReader)
|
|
case "br":
|
|
return io.ReadAll(brotli.NewReader(bytes.NewReader(data)))
|
|
}
|
|
|
|
return data, nil
|
|
}
|
|
|
|
// ReadBodyString 读取响应体为字符串
|
|
func ReadBodyString(resp *http.Response) (string, error) {
|
|
body, err := ReadBody(resp)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return string(body), nil
|
|
}
|
|
|
|
// GetFingerprintInfo 获取指纹信息字符串（用于日志输出）
|
|
func (c *TLSClient) GetFingerprintInfo() string {
|
|
fp := c.fingerprint
|
|
typeName := "tls-client"
|
|
if c.clientType == ClientTypeAzureTLS {
|
|
typeName = "azuretls"
|
|
}
|
|
return fmt.Sprintf("%s/%s %s (%s)", typeName, fp.Browser, fp.Version, fp.Platform)
|
|
}
|
|
|
|
// Close 关闭客户端
|
|
func (c *TLSClient) Close() {
|
|
if c.azureSession != nil {
|
|
c.azureSession.Close()
|
|
}
|
|
}
|