go.mod

@@ -9,15 +9,17 @@ require (
 	github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 // indirect
 	github.com/chromedp/chromedp v0.9.5 // indirect
 	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 )
+
+require github.com/fsnotify/fsnotify v1.7.0 // indirect

go.sum

@@ -8,6 +8,8 @@ github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93
 github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y=
 github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic=
 github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=

main.go

@@ -135,9 +135,14 @@ func runServer() {
 	})
 	initializeTorrentSites()
 
+	http.HandleFunc("/node", handleNodeRequest) // Handle node requests
+
 	config := loadConfig()
 	generateOpenSearchXML(config)
 
+	// Start node communication client
+	go startNodeClient(peers)
+
 	fmt.Printf("Server is listening on http://localhost:%d\n", config.Port)
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", config.Port), nil))
 }

node.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	connCode = "secretcode123"
+	testMsg  = "This is a test message"
+)
+
+var connCodeMutex sync.Mutex
+var peers = []string{"localhost:50002", "localhost:5000"} // Example peer addresses
+
+func handleNodeRequest(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "Invalid request method", http.StatusMethodNotAllowed)
+		return
+	}
+
+	connCodeMutex.Lock()
+	defer connCodeMutex.Unlock()
+
+	body, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		http.Error(w, "Error reading request body", http.StatusInternalServerError)
+		return
+	}
+	defer r.Body.Close()
+
+	receivedCode := strings.TrimSpace(string(body))
+
+	if receivedCode != connCode {
+		http.Error(w, "Authentication failed", http.StatusUnauthorized)
+		return
+	}
+
+	fmt.Println("Authentication successful")
+	fmt.Fprintln(w, testMsg)
+}
+
+func startNodeClient(addresses []string) {
+	for _, address := range addresses {
+		go func(addr string) {
+			for {
+				url := fmt.Sprintf("http://%s/node", addr)
+				req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(connCode))
+				if err != nil {
+					fmt.Println("Error creating request:", err)
+					time.Sleep(5 * time.Second)
+					continue
+				}
+
+				client := &http.Client{}
+				resp, err := client.Do(req)
+				if err != nil {
+					fmt.Println("Error connecting to", addr, ":", err)
+					time.Sleep(5 * time.Second)
+					continue
+				}
+
+				if resp.StatusCode != http.StatusOK {
+					fmt.Println("Authentication failed:", resp.Status)
+					resp.Body.Close()
+					time.Sleep(5 * time.Second)
+					continue
+				}
+
+				body, err := ioutil.ReadAll(resp.Body)
+				if err != nil {
+					fmt.Println("Error reading response:", err)
+					resp.Body.Close()
+					time.Sleep(5 * time.Second)
+					continue
+				}
+				resp.Body.Close()
+
+				testMsg := strings.TrimSpace(string(body))
+				fmt.Println("Received test message from", addr, ":", testMsg)
+				time.Sleep(10 * time.Second)
+			}
+		}(address)
+	}
+}

run.sh

@@ -1,7 +1,35 @@
 #!/bin/sh
 
-# Find all .go files in the current directory
+# Explicitly list the main files in the required order
-GO_FILES=$(find . -name '*.go' -print)
+FILES="
+./main.go
+./init.go
+./search-engine.go
+./text.go
+./text-google.go
+./text-librex.go
+./text-brave.go
+./text-duckduckgo.go
+./common.go
+./cache.go
+./agent.go
+./files.go
+./files-thepiratebay.go
+./files-torrentgalaxy.go
+./forums.go
+./get-searchxng.go
+./imageproxy.go
+./images.go
+./images-imgur.go
+./images-quant.go
+./map.go
+./node.go
+./open-search.go
+./video.go
+"
 
-# Run the Go program
+# Find all other .go files that were not explicitly listed
-go run $GO_FILES
+OTHER_GO_FILES=$(find . -name '*.go' ! -name 'main.go' ! -name 'init.go' ! -name 'search-engine.go' ! -name 'text.go' ! -name 'text-google.go' ! -name 'text-librex.go' ! -name 'text-brave.go' ! -name 'text-duckduckgo.go' ! -name 'common.go' ! -name 'cache.go' ! -name 'agent.go' ! -name 'files.go' ! -name 'files-thepiratebay.go' ! -name 'files-torrentgalaxy.go' ! -name 'forums.go' ! -name 'get-searchxng.go' ! -name 'imageproxy.go' ! -name 'images.go' ! -name 'images-imgur.go' ! -name 'images-quant.go' ! -name 'map.go' ! -name 'node.go' ! -name 'open-search.go' ! -name 'video.go' -print)
+
+# Run the Go program with the specified files first, followed by the remaining files
+go run $FILES $OTHER_GO_FILES

search-engine.go

@@ -1,13 +1,18 @@
 package main
 
 import (
+	"encoding/json"
+	"fmt"
+	"log"
 	"math/rand"
+	"net/http"
 	"sync"
 	"time"
 )
 
 var (
 	searchEngineLock sync.Mutex
+	searchEngines    []SearchEngine // Ensure this variable is defined
 )
 
 // SearchEngine struct now includes metrics for calculating reputation.
@@ -19,11 +24,23 @@ type SearchEngine struct {
 	TotalTime          time.Duration
 	SuccessfulSearches int
 	FailedSearches     int
+	IsCrawler          bool   // Indicates if this search engine is a crawler
+	Host               string // Host of the crawler
+	Port               int    // Port of the crawler
+	AuthCode           string // Auth code for the crawler
 }
 
 // init function seeds the random number generator.
 func init() {
 	rand.Seed(time.Now().UnixNano())
+	// Initialize the searchEngines list
+	searchEngines = []SearchEngine{
+		{Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch), Weight: 1},
+		{Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch), Weight: 2},
+		{Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch), Weight: 2},
+		{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch), Weight: 5},
+		// {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXNGTextSearch), Weight: 2}, // Uncomment when implemented
+	}
 }
 
 // Selects a search engine based on weighted random selection with dynamic weighting.
@@ -88,3 +105,47 @@ func calculateReputation(engine SearchEngine) int {
 	// Scale reputation for better interpretability (e.g., multiply by 10)
 	return int(reputation * 10)
 }
+
+func fetchSearchResults(query, safe, lang, searchType string, page int) []SearchResult {
+	var results []SearchResult
+
+	engine := selectSearchEngine(searchEngines)
+	log.Printf("Using search engine: %s", engine.Name)
+
+	if engine.IsCrawler {
+		searchResults, duration, err := fetchSearchFromCrawler(engine, query, safe, lang, searchType, page)
+		updateEngineMetrics(&engine, duration, err == nil)
+		if err != nil {
+			log.Printf("Error performing search with crawler %s: %v", engine.Name, err)
+			return nil
+		}
+		results = append(results, searchResults...)
+	} else {
+		searchResults, duration, err := engine.Func(query, safe, lang, page)
+		updateEngineMetrics(&engine, duration, err == nil)
+		if err != nil {
+			log.Printf("Error performing search with %s: %v", engine.Name, err)
+			return nil
+		}
+		results = append(results, searchResults...)
+	}
+
+	return results
+}
+
+func fetchSearchFromCrawler(engine SearchEngine, query, safe, lang, searchType string, page int) ([]SearchResult, time.Duration, error) {
+	url := fmt.Sprintf("http://%s:%d/search?q=%s&safe=%s&lang=%s&t=%s&p=%d", engine.Host, engine.Port, query, safe, lang, searchType, page)
+	start := time.Now()
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	var results []SearchResult
+	if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
+		return nil, 0, err
+	}
+
+	return results, time.Since(start), nil
+}

text.go

@@ -15,7 +15,7 @@ func init() {
 		{Name: "Google", Func: wrapTextSearchFunc(PerformGoogleTextSearch), Weight: 1},
 		{Name: "LibreX", Func: wrapTextSearchFunc(PerformLibreXTextSearch), Weight: 2},
 		{Name: "Brave", Func: wrapTextSearchFunc(PerformBraveTextSearch), Weight: 2},
-		{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch), Weight: 5}, // DuckDuckGo timeouts too fast and search results are trash
+		{Name: "DuckDuckGo", Func: wrapTextSearchFunc(PerformDuckDuckGoTextSearch), Weight: 5},
 		// {Name: "SearXNG", Func: wrapTextSearchFunc(PerformSearXNGTextSearch), Weight: 2}, // Uncomment when implemented
 	}
 }