From e2893967cd1812757cfa0996d1e4d9d8ca86e8c8 Mon Sep 17 00:00:00 2001 From: partisan Date: Sun, 2 Jun 2024 12:05:25 +0200 Subject: [PATCH] random user agents --- agent.go | 355 +++++++++++++++++++++++++++++++++++++++++ files-thepiratebay.go | 17 +- files-torrentgalaxy.go | 17 +- images.go | 13 +- run.sh | 2 +- text-google.go | 13 +- text.go | 6 +- video.go | 7 +- 8 files changed, 418 insertions(+), 12 deletions(-) create mode 100644 agent.go diff --git a/agent.go b/agent.go new file mode 100644 index 0000000..172b209 --- /dev/null +++ b/agent.go @@ -0,0 +1,355 @@ +package main + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "math/rand" + "net/http" + "sort" + "sync" + "time" +) + +type BrowserVersion struct { + Version string `json:"version"` + Global float64 `json:"global"` +} + +type BrowserData struct { + Firefox []BrowserVersion `json:"firefox"` + Chromium []BrowserVersion `json:"chrome"` +} + +var ( + cache = struct { + sync.RWMutex + data map[string]string + }{ + data: make(map[string]string), + } + browserCache = struct { + sync.RWMutex + data BrowserData + expires time.Time + }{ + expires: time.Now(), + } +) + +func fetchLatestBrowserVersions() (BrowserData, error) { + url := "https://raw.githubusercontent.com/Fyrd/caniuse/master/fulldata-json/data-2.0.json" + + resp, err := http.Get(url) + if err != nil { + return BrowserData{}, err + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return BrowserData{}, err + } + + var rawData map[string]interface{} + if err := json.Unmarshal(body, &rawData); err != nil { + return BrowserData{}, err + } + + stats := rawData["agents"].(map[string]interface{}) + + var data BrowserData + + if firefoxData, ok := stats["firefox"].(map[string]interface{}); ok { + for version, usage := range firefoxData["usage_global"].(map[string]interface{}) { + data.Firefox = append(data.Firefox, BrowserVersion{ + Version: version, + Global: usage.(float64), + }) + } + } + + if chromeData, ok := stats["chrome"].(map[string]interface{}); ok { + for version, usage := range chromeData["usage_global"].(map[string]interface{}) { + data.Chromium = append(data.Chromium, BrowserVersion{ + Version: version, + Global: usage.(float64), + }) + } + } + + return data, nil +} + +func getLatestBrowserVersions() (BrowserData, error) { + browserCache.RLock() + if time.Now().Before(browserCache.expires) { + data := browserCache.data + browserCache.RUnlock() + return data, nil + } + browserCache.RUnlock() + + data, err := fetchLatestBrowserVersions() + if err != nil { + return BrowserData{}, err + } + + browserCache.Lock() + browserCache.data = data + browserCache.expires = time.Now().Add(24 * time.Hour) + browserCache.Unlock() + + return data, nil +} + +func randomUserAgent() (string, error) { + browsers, err := getLatestBrowserVersions() + if err != nil { + return "", err + } + + rand.Seed(time.Now().UnixNano()) + + // Simulated browser usage statistics (in percentages) + usageStats := map[string]float64{ + "Firefox": 30.0, + "Chromium": 70.0, + } + + // Calculate the probabilities for the versions + probabilities := []float64{0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625} + + // Select a browser based on usage statistics + browserType := "" + randVal := rand.Float64() * 100 + cumulative := 0.0 + for browser, usage := range usageStats { + cumulative += usage + if randVal < cumulative { + browserType = browser + break + } + } + + var versions []BrowserVersion + switch browserType { + case "Firefox": + versions = browsers.Firefox + case "Chromium": + versions = browsers.Chromium + } + + if len(versions) == 0 { + return "", fmt.Errorf("no versions found for browser: %s", browserType) + } + + // Sort versions by usage (descending order) + sort.Slice(versions, func(i, j int) bool { + return versions[i].Global > versions[j].Global + }) + + // Select a version based on the probabilities + version := "" + randVal = rand.Float64() + cumulative = 0.0 + for i, p := range probabilities { + cumulative += p + if randVal < cumulative && i < len(versions) { + version = versions[i].Version + break + } + } + + if version == "" { + version = versions[len(versions)-1].Version + } + + // Generate the user agent string + userAgent := generateUserAgent(browserType, version) + return userAgent, nil +} + +func generateUserAgent(browser, version string) string { + oses := []struct { + os string + probability float64 + }{ + {"Windows NT 10.0; Win64; x64", 44.0}, + {"Windows NT 11.0; Win64; x64", 44.0}, + {"X11; Linux x86_64", 1.0}, + {"X11; Ubuntu; Linux x86_64", 1.0}, + {"Macintosh; Intel Mac OS X 10_15_7", 10.0}, + } + + // Select an OS based on probabilities + randVal := rand.Float64() * 100 + cumulative := 0.0 + selectedOS := "" + for _, os := range oses { + cumulative += os.probability + if randVal < cumulative { + selectedOS = os.os + break + } + } + + switch browser { + case "Firefox": + return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s", selectedOS, version, version) + case "Chromium": + return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", selectedOS, version) + } + return "" +} + +func updateCachedUserAgents(newVersions BrowserData) { + cache.Lock() + defer cache.Unlock() + for key, userAgent := range cache.data { + randVal := rand.Float64() + if randVal < 0.5 { + updatedUserAgent := updateUserAgentVersion(userAgent, newVersions) + cache.data[key] = updatedUserAgent + } + } +} + +func updateUserAgentVersion(userAgent string, newVersions BrowserData) string { + // Parse the current user agent to extract browser and version + var browserType, version string + if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { + browserType = "Chromium" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { + browserType = "Chromium" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { + browserType = "Chromium" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { + browserType = "Chromium" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil { + browserType = "Chromium" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { + browserType = "Firefox" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 11.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { + browserType = "Firefox" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { + browserType = "Firefox" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { + browserType = "Firefox" + } else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil { + browserType = "Firefox" + } + + // Get the latest version for the browser type + var latestVersion string + if browserType == "Firefox" { + latestVersion = newVersions.Firefox[0].Version + } else if browserType == "Chromium" { + latestVersion = newVersions.Chromium[0].Version + } + + // Update the user agent string with the new version + return generateUserAgent(browserType, latestVersion) +} + +func periodicUpdate() { + for { + // Sleep for a random interval between 1 and 2 days + time.Sleep(time.Duration(24+rand.Intn(24)) * time.Hour) + + // Fetch the latest browser versions + newVersions, err := fetchLatestBrowserVersions() + if err != nil { + fmt.Println("Error fetching latest browser versions:", err) + continue + } + + // Update the browser version cache + browserCache.Lock() + browserCache.data = newVersions + browserCache.expires = time.Now().Add(24 * time.Hour) + browserCache.Unlock() + + // Update the cached user agents + updateCachedUserAgents(newVersions) + } +} + +func GetUserAgent(cacheKey string) (string, error) { + cache.RLock() + userAgent, found := cache.data[cacheKey] + cache.RUnlock() + + if found { + return userAgent, nil + } + + userAgent, err := randomUserAgent() + if err != nil { + return "", err + } + + cache.Lock() + cache.data[cacheKey] = userAgent + cache.Unlock() + + return userAgent, nil +} + +func GetNewUserAgent(cacheKey string) (string, error) { + userAgent, err := randomUserAgent() + if err != nil { + return "", err + } + + cache.Lock() + cache.data[cacheKey] = userAgent + cache.Unlock() + + return userAgent, nil +} + +func init() { + go periodicUpdate() +} + +// func main() { +// go periodicUpdate() // not needed here + +// cacheKey := "image-search" +// userAgent, err := GetUserAgent(cacheKey) +// if err != nil { +// fmt.Println("Error:", err) +// return +// } + +// fmt.Println("Generated User Agent:", userAgent) + +// // Request a new user agent for the same key +// newUserAgent, err := GetNewUserAgent(cacheKey) +// if err != nil { +// fmt.Println("Error:", err) +// return +// } + +// fmt.Println("New User Agent:", newUserAgent) + +// AcacheKey := "image-search" +// AuserAgent, err := GetUserAgent(AcacheKey) +// if err != nil { +// fmt.Println("Error:", err) +// return +// } + +// fmt.Println("Generated User Agent:", AuserAgent) + +// DcacheKey := "image-search" +// DuserAgent, err := GetUserAgent(DcacheKey) +// if err != nil { +// fmt.Println("Error:", err) +// return +// } + +// fmt.Println("Generated User Agent:", DuserAgent) + +// } diff --git a/files-thepiratebay.go b/files-thepiratebay.go index 07514f5..e88d6f3 100644 --- a/files-thepiratebay.go +++ b/files-thepiratebay.go @@ -58,7 +58,22 @@ func (t *ThePirateBay) Search(query string, category string) ([]TorrentResult, e } url := fmt.Sprintf("https://%s/q.php?q=%s&cat=%s", PIRATEBAY_DOMAIN, url.QueryEscape(query), categoryCode) - response, err := http.Get(url) + + // User Agent generation + userAgent, err := GetUserAgent("files-tpb") + if err != nil { + fmt.Println("Error:", err) + return nil, err + } + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + + client := &http.Client{} + response, err := client.Do(req) if err != nil { return nil, err } diff --git a/files-torrentgalaxy.go b/files-torrentgalaxy.go index cb5db03..78f08bd 100644 --- a/files-torrentgalaxy.go +++ b/files-torrentgalaxy.go @@ -58,7 +58,22 @@ func (tg *TorrentGalaxy) Search(query string, category string) ([]TorrentResult, } searchURL := fmt.Sprintf("https://%s/torrents.php?search=%s%s#results", TORRENTGALAXY_DOMAIN, url.QueryEscape(query), categoryCode) - resp, err := http.Get(searchURL) + + // User Agent generation + userAgent, err := GetUserAgent("files-torrentgalaxy") + if err != nil { + fmt.Println("Error:", err) + return nil, err + } + + req, err := http.NewRequest("GET", searchURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", userAgent) + + client := &http.Client{} + resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("error making request to TorrentGalaxy: %w", err) } diff --git a/images.go b/images.go index 41586f8..16e8581 100644 --- a/images.go +++ b/images.go @@ -70,7 +70,18 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch return nil, fmt.Errorf("creating request: %v", err) } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") + // User Agent generation + ImageUserAgent, err := GetUserAgent("Image-Search") + if err != nil { + fmt.Println("Error:", err) + return nil, err + } + + if debugMode { + fmt.Println("Generated User Agent (images):", ImageUserAgent) + } + + req.Header.Set("User-Agent", ImageUserAgent) resp, err := client.Do(req) if err != nil { diff --git a/run.sh b/run.sh index 566c848..60eb68e 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go --debug \ No newline at end of file +go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug \ No newline at end of file diff --git a/text-google.go b/text-google.go index 760128c..9c338cc 100644 --- a/text-google.go +++ b/text-google.go @@ -22,7 +22,18 @@ func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchRe return nil, fmt.Errorf("failed to create request: %v", err) } - req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36") + // User Agent generation + TextUserAgent, err := GetUserAgent("Text-Search") + if err != nil { + fmt.Println("Error:", err) + return nil, err + } + + if debugMode { + fmt.Println("Generated User Agent (text):", TextUserAgent) + } + + req.Header.Set("User-Agent", TextUserAgent) resp, err := client.Do(req) if err != nil { diff --git a/text.go b/text.go index 31d36af..b65f57c 100644 --- a/text.go +++ b/text.go @@ -136,7 +136,7 @@ func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int }{ {PerformGoogleTextSearch, "Google"}, {PerformDuckDuckGoTextSearch, "DuckDuckGo"}, - {PerformQwantTextSearch, "Qwant"}, + // {PerformQwantTextSearch, "Qwant"}, } wg.Add(len(searchFuncs)) @@ -182,8 +182,8 @@ func sourceOrder(source string) int { return 1 case "DuckDuckGo": return 2 - case "Qwant": - return 3 + // case "Qwant": + // return 3 default: return 4 } diff --git a/video.go b/video.go index b64d2cb..ec78793 100644 --- a/video.go +++ b/video.go @@ -117,7 +117,7 @@ func testInstanceAvailability(instance string) bool { return true } -func makeHTMLRequest(query string) (*VideoAPIResponse, error) { +func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) { var lastError error mu.Lock() defer mu.Unlock() @@ -127,7 +127,7 @@ func makeHTMLRequest(query string) (*VideoAPIResponse, error) { continue // Skip this instance because it's still disabled } - url := fmt.Sprintf("https://%s/search?q=%s&filter=all", instance, url.QueryEscape(query)) + url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d", instance, url.QueryEscape(query), safe, lang, page) resp, err := http.Get(url) if err != nil || resp.StatusCode != http.StatusOK { log.Printf("Disabling instance %s due to error or status code: %v", instance, err) @@ -151,8 +151,7 @@ func makeHTMLRequest(query string) (*VideoAPIResponse, error) { func handleVideoSearch(w http.ResponseWriter, query, safe, lang string, page int) { start := time.Now() - // Modify `makeHTMLRequest` to also accept `safe`, `lang`, and `page` parameters if necessary - apiResp, err := makeHTMLRequest(query) + apiResp, err := makeHTMLRequest(query, safe, lang, page) if err != nil { log.Printf("Error fetching video results: %v", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError)