random user agents

agent.go

@@ -0,0 +1,355 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"sort"
+	"sync"
+	"time"
+)
+
+type BrowserVersion struct {
+	Version string  `json:"version"`
+	Global  float64 `json:"global"`
+}
+
+type BrowserData struct {
+	Firefox  []BrowserVersion `json:"firefox"`
+	Chromium []BrowserVersion `json:"chrome"`
+}
+
+var (
+	cache = struct {
+		sync.RWMutex
+		data map[string]string
+	}{
+		data: make(map[string]string),
+	}
+	browserCache = struct {
+		sync.RWMutex
+		data    BrowserData
+		expires time.Time
+	}{
+		expires: time.Now(),
+	}
+)
+
+func fetchLatestBrowserVersions() (BrowserData, error) {
+	url := "https://raw.githubusercontent.com/Fyrd/caniuse/master/fulldata-json/data-2.0.json"
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return BrowserData{}, err
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return BrowserData{}, err
+	}
+
+	var rawData map[string]interface{}
+	if err := json.Unmarshal(body, &rawData); err != nil {
+		return BrowserData{}, err
+	}
+
+	stats := rawData["agents"].(map[string]interface{})
+
+	var data BrowserData
+
+	if firefoxData, ok := stats["firefox"].(map[string]interface{}); ok {
+		for version, usage := range firefoxData["usage_global"].(map[string]interface{}) {
+			data.Firefox = append(data.Firefox, BrowserVersion{
+				Version: version,
+				Global:  usage.(float64),
+			})
+		}
+	}
+
+	if chromeData, ok := stats["chrome"].(map[string]interface{}); ok {
+		for version, usage := range chromeData["usage_global"].(map[string]interface{}) {
+			data.Chromium = append(data.Chromium, BrowserVersion{
+				Version: version,
+				Global:  usage.(float64),
+			})
+		}
+	}
+
+	return data, nil
+}
+
+func getLatestBrowserVersions() (BrowserData, error) {
+	browserCache.RLock()
+	if time.Now().Before(browserCache.expires) {
+		data := browserCache.data
+		browserCache.RUnlock()
+		return data, nil
+	}
+	browserCache.RUnlock()
+
+	data, err := fetchLatestBrowserVersions()
+	if err != nil {
+		return BrowserData{}, err
+	}
+
+	browserCache.Lock()
+	browserCache.data = data
+	browserCache.expires = time.Now().Add(24 * time.Hour)
+	browserCache.Unlock()
+
+	return data, nil
+}
+
+func randomUserAgent() (string, error) {
+	browsers, err := getLatestBrowserVersions()
+	if err != nil {
+		return "", err
+	}
+
+	rand.Seed(time.Now().UnixNano())
+
+	// Simulated browser usage statistics (in percentages)
+	usageStats := map[string]float64{
+		"Firefox":  30.0,
+		"Chromium": 70.0,
+	}
+
+	// Calculate the probabilities for the versions
+	probabilities := []float64{0.5, 0.25, 0.125, 0.0625, 0.03125, 0.015625, 0.0078125, 0.00390625}
+
+	// Select a browser based on usage statistics
+	browserType := ""
+	randVal := rand.Float64() * 100
+	cumulative := 0.0
+	for browser, usage := range usageStats {
+		cumulative += usage
+		if randVal < cumulative {
+			browserType = browser
+			break
+		}
+	}
+
+	var versions []BrowserVersion
+	switch browserType {
+	case "Firefox":
+		versions = browsers.Firefox
+	case "Chromium":
+		versions = browsers.Chromium
+	}
+
+	if len(versions) == 0 {
+		return "", fmt.Errorf("no versions found for browser: %s", browserType)
+	}
+
+	// Sort versions by usage (descending order)
+	sort.Slice(versions, func(i, j int) bool {
+		return versions[i].Global > versions[j].Global
+	})
+
+	// Select a version based on the probabilities
+	version := ""
+	randVal = rand.Float64()
+	cumulative = 0.0
+	for i, p := range probabilities {
+		cumulative += p
+		if randVal < cumulative && i < len(versions) {
+			version = versions[i].Version
+			break
+		}
+	}
+
+	if version == "" {
+		version = versions[len(versions)-1].Version
+	}
+
+	// Generate the user agent string
+	userAgent := generateUserAgent(browserType, version)
+	return userAgent, nil
+}
+
+func generateUserAgent(browser, version string) string {
+	oses := []struct {
+		os          string
+		probability float64
+	}{
+		{"Windows NT 10.0; Win64; x64", 44.0},
+		{"Windows NT 11.0; Win64; x64", 44.0},
+		{"X11; Linux x86_64", 1.0},
+		{"X11; Ubuntu; Linux x86_64", 1.0},
+		{"Macintosh; Intel Mac OS X 10_15_7", 10.0},
+	}
+
+	// Select an OS based on probabilities
+	randVal := rand.Float64() * 100
+	cumulative := 0.0
+	selectedOS := ""
+	for _, os := range oses {
+		cumulative += os.probability
+		if randVal < cumulative {
+			selectedOS = os.os
+			break
+		}
+	}
+
+	switch browser {
+	case "Firefox":
+		return fmt.Sprintf("Mozilla/5.0 (%s; rv:%s) Gecko/20100101 Firefox/%s", selectedOS, version, version)
+	case "Chromium":
+		return fmt.Sprintf("Mozilla/5.0 (%s) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", selectedOS, version)
+	}
+	return ""
+}
+
+func updateCachedUserAgents(newVersions BrowserData) {
+	cache.Lock()
+	defer cache.Unlock()
+	for key, userAgent := range cache.data {
+		randVal := rand.Float64()
+		if randVal < 0.5 {
+			updatedUserAgent := updateUserAgentVersion(userAgent, newVersions)
+			cache.data[key] = updatedUserAgent
+		}
+	}
+}
+
+func updateUserAgentVersion(userAgent string, newVersions BrowserData) string {
+	// Parse the current user agent to extract browser and version
+	var browserType, version string
+	if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
+		browserType = "Chromium"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
+		browserType = "Chromium"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
+		browserType = "Chromium"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
+		browserType = "Chromium"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36", &version); err == nil {
+		browserType = "Chromium"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
+		browserType = "Firefox"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Windows NT 11.0; Win64; x64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
+		browserType = "Firefox"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
+		browserType = "Firefox"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
+		browserType = "Firefox"
+	} else if _, err := fmt.Sscanf(userAgent, "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7; rv:%s) Gecko/20100101 Firefox/%s", &version, &version); err == nil {
+		browserType = "Firefox"
+	}
+
+	// Get the latest version for the browser type
+	var latestVersion string
+	if browserType == "Firefox" {
+		latestVersion = newVersions.Firefox[0].Version
+	} else if browserType == "Chromium" {
+		latestVersion = newVersions.Chromium[0].Version
+	}
+
+	// Update the user agent string with the new version
+	return generateUserAgent(browserType, latestVersion)
+}
+
+func periodicUpdate() {
+	for {
+		// Sleep for a random interval between 1 and 2 days
+		time.Sleep(time.Duration(24+rand.Intn(24)) * time.Hour)
+
+		// Fetch the latest browser versions
+		newVersions, err := fetchLatestBrowserVersions()
+		if err != nil {
+			fmt.Println("Error fetching latest browser versions:", err)
+			continue
+		}
+
+		// Update the browser version cache
+		browserCache.Lock()
+		browserCache.data = newVersions
+		browserCache.expires = time.Now().Add(24 * time.Hour)
+		browserCache.Unlock()
+
+		// Update the cached user agents
+		updateCachedUserAgents(newVersions)
+	}
+}
+
+func GetUserAgent(cacheKey string) (string, error) {
+	cache.RLock()
+	userAgent, found := cache.data[cacheKey]
+	cache.RUnlock()
+
+	if found {
+		return userAgent, nil
+	}
+
+	userAgent, err := randomUserAgent()
+	if err != nil {
+		return "", err
+	}
+
+	cache.Lock()
+	cache.data[cacheKey] = userAgent
+	cache.Unlock()
+
+	return userAgent, nil
+}
+
+func GetNewUserAgent(cacheKey string) (string, error) {
+	userAgent, err := randomUserAgent()
+	if err != nil {
+		return "", err
+	}
+
+	cache.Lock()
+	cache.data[cacheKey] = userAgent
+	cache.Unlock()
+
+	return userAgent, nil
+}
+
+func init() {
+	go periodicUpdate()
+}
+
+// func main() {
+// 	go periodicUpdate() // not needed here
+
+// 	cacheKey := "image-search"
+// 	userAgent, err := GetUserAgent(cacheKey)
+// 	if err != nil {
+// 		fmt.Println("Error:", err)
+// 		return
+// 	}
+
+// 	fmt.Println("Generated User Agent:", userAgent)
+
+// 	// Request a new user agent for the same key
+// 	newUserAgent, err := GetNewUserAgent(cacheKey)
+// 	if err != nil {
+// 		fmt.Println("Error:", err)
+// 		return
+// 	}
+
+// 	fmt.Println("New User Agent:", newUserAgent)
+
+// 	AcacheKey := "image-search"
+// 	AuserAgent, err := GetUserAgent(AcacheKey)
+// 	if err != nil {
+// 		fmt.Println("Error:", err)
+// 		return
+// 	}
+
+// 	fmt.Println("Generated User Agent:", AuserAgent)
+
+// 	DcacheKey := "image-search"
+// 	DuserAgent, err := GetUserAgent(DcacheKey)
+// 	if err != nil {
+// 		fmt.Println("Error:", err)
+// 		return
+// 	}
+
+// 	fmt.Println("Generated User Agent:", DuserAgent)
+
+// }

files-thepiratebay.go

@@ -58,7 +58,22 @@ func (t *ThePirateBay) Search(query string, category string) ([]TorrentResult, e
 	}
 
 	url := fmt.Sprintf("https://%s/q.php?q=%s&cat=%s", PIRATEBAY_DOMAIN, url.QueryEscape(query), categoryCode)
-	response, err := http.Get(url)
+
+	// User Agent generation
+	userAgent, err := GetUserAgent("files-tpb")
+	if err != nil {
+		fmt.Println("Error:", err)
+		return nil, err
+	}
+
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	client := &http.Client{}
+	response, err := client.Do(req)
 	if err != nil {
 		return nil, err
 	}

files-torrentgalaxy.go

@@ -58,7 +58,22 @@ func (tg *TorrentGalaxy) Search(query string, category string) ([]TorrentResult,
 	}
 
 	searchURL := fmt.Sprintf("https://%s/torrents.php?search=%s%s#results", TORRENTGALAXY_DOMAIN, url.QueryEscape(query), categoryCode)
-	resp, err := http.Get(searchURL)
+
+	// User Agent generation
+	userAgent, err := GetUserAgent("files-torrentgalaxy")
+	if err != nil {
+		fmt.Println("Error:", err)
+		return nil, err
+	}
+
+	req, err := http.NewRequest("GET", searchURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, fmt.Errorf("error making request to TorrentGalaxy: %w", err)
 	}

images.go

@@ -70,7 +70,18 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch
 		return nil, fmt.Errorf("creating request: %v", err)
 	}
 
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
+	// User Agent generation
+	ImageUserAgent, err := GetUserAgent("Image-Search")
+	if err != nil {
+		fmt.Println("Error:", err)
+		return nil, err
+	}
+
+	if debugMode {
+		fmt.Println("Generated User Agent (images):", ImageUserAgent)
+	}
+
+	req.Header.Set("User-Agent", ImageUserAgent)
 
 	resp, err := client.Do(req)
 	if err != nil {

run.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go --debug
+go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go files-thepiratebay.go agent.go --debug

text-google.go

@@ -22,7 +22,18 @@ func PerformGoogleTextSearch(query, safe, lang string, page int) ([]TextSearchRe
 		return nil, fmt.Errorf("failed to create request: %v", err)
 	}
 
-	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36")
+	// User Agent generation
+	TextUserAgent, err := GetUserAgent("Text-Search")
+	if err != nil {
+		fmt.Println("Error:", err)
+		return nil, err
+	}
+
+	if debugMode {
+		fmt.Println("Generated User Agent (text):", TextUserAgent)
+	}
+
+	req.Header.Set("User-Agent", TextUserAgent)
 
 	resp, err := client.Do(req)
 	if err != nil {

text.go

@@ -136,7 +136,7 @@ func fetchAndCacheTextResults(query, safe, lang string, page, resultsPerPage int
 	}{
 		{PerformGoogleTextSearch, "Google"},
 		{PerformDuckDuckGoTextSearch, "DuckDuckGo"},
-		{PerformQwantTextSearch, "Qwant"},
+		// {PerformQwantTextSearch, "Qwant"},
 	}
 
 	wg.Add(len(searchFuncs))
@@ -182,8 +182,8 @@ func sourceOrder(source string) int {
 		return 1
 	case "DuckDuckGo":
 		return 2
-	case "Qwant":
+	// case "Qwant":
-		return 3
+	// 	return 3
 	default:
 		return 4
 	}

video.go

@@ -117,7 +117,7 @@ func testInstanceAvailability(instance string) bool {
 	return true
 }
 
-func makeHTMLRequest(query string) (*VideoAPIResponse, error) {
+func makeHTMLRequest(query, safe, lang string, page int) (*VideoAPIResponse, error) {
 	var lastError error
 	mu.Lock()
 	defer mu.Unlock()
@@ -127,7 +127,7 @@ func makeHTMLRequest(query string) (*VideoAPIResponse, error) {
 			continue // Skip this instance because it's still disabled
 		}
 
-		url := fmt.Sprintf("https://%s/search?q=%s&filter=all", instance, url.QueryEscape(query))
+		url := fmt.Sprintf("https://%s/search?q=%s&filter=all&safe=%s&lang=%s&page=%d", instance, url.QueryEscape(query), safe, lang, page)
 		resp, err := http.Get(url)
 		if err != nil || resp.StatusCode != http.StatusOK {
 			log.Printf("Disabling instance %s due to error or status code: %v", instance, err)
@@ -151,8 +151,7 @@ func makeHTMLRequest(query string) (*VideoAPIResponse, error) {
 func handleVideoSearch(w http.ResponseWriter, query, safe, lang string, page int) {
 	start := time.Now()
 
-	// Modify `makeHTMLRequest` to also accept `safe`, `lang`, and `page` parameters if necessary
+	apiResp, err := makeHTMLRequest(query, safe, lang, page)
-	apiResp, err := makeHTMLRequest(query)
 	if err != nil {
 		log.Printf("Error fetching video results: %v", err)
 		http.Error(w, "Internal Server Error", http.StatusInternalServerError)