From 4ccb9a51d454a13cd8382059cbba5988e51ec65b Mon Sep 17 00:00:00 2001 From: partisan Date: Thu, 23 May 2024 10:56:26 +0200 Subject: [PATCH] files search added (wip) --- README.md | 2 +- files-torrentgalaxy.go | 185 +++++++++++++++++++++++++++++++++++++++++ files.go | 123 +++++++++++++++++++++++++++ forums.go | 7 +- images.go | 2 +- main.go | 6 +- run.sh | 2 +- templates/files.html | 112 +++++++++++++++++++++++++ templates/text.html | 4 +- text-duckduckgo.go | 1 - text-google.go | 1 - text-quant.go | 1 - text.go | 16 ---- video.go | 15 ---- 14 files changed, 432 insertions(+), 45 deletions(-) create mode 100644 files-torrentgalaxy.go create mode 100644 files.go create mode 100644 templates/files.html diff --git a/README.md b/README.md index b41f13d..d752c81 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,10 @@ - [X] Forums results - [x] HTML+CSS site (no JS version) - [X] Results cache +- [X] Torrent results ## Pending Tasks -- [ ] Torrent results - [ ] Website with JS version - [ ] JS applets for results (such as calculator) - [ ] Dynamic results loading as user scrolls diff --git a/files-torrentgalaxy.go b/files-torrentgalaxy.go new file mode 100644 index 0000000..20284e3 --- /dev/null +++ b/files-torrentgalaxy.go @@ -0,0 +1,185 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +const TORRENTGALAXY_DOMAIN = "torrentgalaxy.to" + +type TorrentGalaxy struct{} + +func NewTorrentGalaxy() *TorrentGalaxy { + return &TorrentGalaxy{} +} + +func (tg *TorrentGalaxy) Name() string { + return "torrentgalaxy" +} + +func (tg *TorrentGalaxy) getCategoryCode(category string) string { + switch category { + case "all": + return "" + case "audiobook": + return "&c13=1" + case "movie": + return "&c3=1&c46=1&c45=1&c42=1&c4=1&c1=1" + case "tv": + return "&c41=1&c5=1&c11=1&c6=1&c7=1" + case "games": + return "&c43=1&c10=1" + case "software": + return "&c20=1&c21=1&c18=1" + case "anime": + return "&c28=1" + case "music": + return "&c28=1&c22=1&c26=1&c23=1&c25=1&c24=1" + case "xxx": + safeSearch := true // Replace with actual safe search status + if safeSearch { + return "ignore" + } + return "&c48=1&c35=1&c47=1&c34=1" + default: + return "" + } +} + +func (tg *TorrentGalaxy) Search(query string, category string) ([]TorrentResult, error) { + categoryCode := tg.getCategoryCode(category) + if categoryCode == "ignore" { + return []TorrentResult{}, nil + } + + searchURL := fmt.Sprintf("https://%s/torrents.php?search=%s%s#results", TORRENTGALAXY_DOMAIN, url.QueryEscape(query), categoryCode) + resp, err := http.Get(searchURL) + if err != nil { + return nil, fmt.Errorf("error making request to TorrentGalaxy: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("error parsing HTML: %w", err) + } + + var results []TorrentResult + doc.Find("div.tgxtablerow").Each(func(i int, s *goquery.Selection) { + titleDiv := s.Find("div#click") + title := strings.TrimSpace(titleDiv.Text()) + magnetLink, exists := s.Find("a[href^='magnet']").Attr("href") + if !exists { + log.Printf("No magnet link found for title: %s", title) + return + } + + byteSize := parseSize(s.Find("span.badge-secondary").Text()) + viewCount := parseInt(s.Find("font[color='orange']").Text()) + seeder := parseInt(s.Find("font[color='green']").Text()) + leecher := parseInt(s.Find("font[color='#ff0000']").Text()) + + result := TorrentResult{ + URL: fmt.Sprintf("https://%s", TORRENTGALAXY_DOMAIN), + Seeders: seeder, + Leechers: leecher, + Magnet: applyTrackers(magnetLink), + Views: viewCount, + Size: formatSize(byteSize), + Title: title, + } + results = append(results, result) + }) + + return results, nil +} + +func parseInt(s string) int { + s = strings.ReplaceAll(s, ",", "") + result, err := strconv.Atoi(s) + if err != nil { + log.Printf("Error parsing integer: %v", err) + return 0 + } + return result +} + +func parseSize(sizeStr string) int64 { + sizeStr = strings.TrimSpace(sizeStr) + if sizeStr == "" { + return 0 + } + + // Use regex to extract numeric value and unit separately + re := regexp.MustCompile(`(?i)([\d.]+)\s*([KMGT]?B)`) + matches := re.FindStringSubmatch(sizeStr) + if len(matches) < 3 { + log.Printf("Error parsing size: invalid format %s", sizeStr) + return 0 + } + + sizeStr = matches[1] + unit := strings.ToUpper(matches[2]) + + var multiplier int64 = 1 + switch unit { + case "KB": + multiplier = 1024 + case "MB": + multiplier = 1024 * 1024 + case "GB": + multiplier = 1024 * 1024 * 1024 + case "TB": + multiplier = 1024 * 1024 * 1024 * 1024 + default: + log.Printf("Unknown unit: %s", unit) + return 0 + } + + size, err := strconv.ParseFloat(sizeStr, 64) + if err != nil { + log.Printf("Error parsing size: %v", err) + return 0 + } + return int64(size * float64(multiplier)) +} + +func applyTrackers(magnetLink string) string { + if magnetLink == "" { + return "" + } + trackers := []string{ + "udp://tracker.openbittorrent.com:80/announce", + "udp://tracker.opentrackr.org:1337/announce", + "udp://tracker.coppersurfer.tk:6969/announce", + "udp://tracker.leechers-paradise.org:6969/announce", + } + for _, tracker := range trackers { + magnetLink += "&tr=" + url.QueryEscape(tracker) + } + return magnetLink +} + +func formatSize(size int64) string { + if size >= 1024*1024*1024*1024 { + return fmt.Sprintf("%.2f TB", float64(size)/(1024*1024*1024*1024)) + } else if size >= 1024*1024*1024 { + return fmt.Sprintf("%.2f GB", float64(size)/(1024*1024*1024)) + } else if size >= 1024*1024 { + return fmt.Sprintf("%.2f MB", float64(size)/(1024*1024)) + } else if size >= 1024 { + return fmt.Sprintf("%.2f KB", float64(size)/1024) + } + return fmt.Sprintf("%d B", size) +} diff --git a/files.go b/files.go new file mode 100644 index 0000000..dc7baf7 --- /dev/null +++ b/files.go @@ -0,0 +1,123 @@ +package main + +import ( + "fmt" + "html/template" + "log" + "net/http" + "sort" + "time" +) + +type Settings struct { + UxLang string + Safe string +} + +type TorrentSite interface { + Name() string + Search(query string, category string) ([]TorrentResult, error) +} + +type TorrentResult struct { + URL string + Seeders int + Leechers int + Magnet string + Views int + Size string + Title string + Error string +} + +var ( + torrentGalaxy TorrentSite + nyaa TorrentSite + thePirateBay TorrentSite + rutor TorrentSite +) + +func initializeTorrentSites() { + torrentGalaxy = NewTorrentGalaxy() + // nyaa = NewNyaa() + // thePirateBay = NewThePirateBay() + // rutor = NewRutor() +} + +func handleFileSearch(w http.ResponseWriter, query, safe, lang string, page int) { + startTime := time.Now() + + settings := Settings{UxLang: lang, Safe: safe} + sites := []TorrentSite{torrentGalaxy, nyaa, thePirateBay, rutor} + results := []TorrentResult{} + allErrors := true + + for _, site := range sites { + if site == nil { + continue + } + res, err := site.Search(query, "all") + if err != nil { + continue + } + if len(res) > 0 { + allErrors = false + } + results = append(results, res...) + } + + if allErrors { + results = []TorrentResult{ + {Error: "Results are currently unavailable, sorry. Please try again later."}, + } + } + + sort.Slice(results, func(i, j int) bool { return results[i].Seeders > results[j].Seeders }) + + elapsedTime := time.Since(startTime) + funcMap := template.FuncMap{ + "sub": subtract, + "add": add, + } + tmpl, err := template.New("files.html").Funcs(funcMap).ParseFiles("templates/files.html") + if err != nil { + log.Printf("Failed to load template: %v", err) + http.Error(w, "Failed to load template", http.StatusInternalServerError) + return + } + + data := struct { + Results []TorrentResult + Query string + Fetched string + Category string + Sort string + HasPrevPage bool + HasNextPage bool + Page int + Settings Settings + }{ + Results: results, + Query: query, + Fetched: fmt.Sprintf("%.2f", elapsedTime.Seconds()), + Category: "all", + Sort: "seed", + HasPrevPage: page > 1, + HasNextPage: len(results) > 0, + Page: page, + Settings: settings, + } + + if err := tmpl.Execute(w, data); err != nil { + log.Printf("Failed to render template: %v", err) + http.Error(w, "Failed to render template", http.StatusInternalServerError) + } +} + +func subtract(a, b int) int { + return a - b +} + +func add(a, b int) int { + return a + b +} diff --git a/forums.go b/forums.go index 348bc6c..be2d6a0 100644 --- a/forums.go +++ b/forums.go @@ -1,4 +1,3 @@ -// forums.go package main import ( @@ -23,13 +22,13 @@ type ForumSearchResult struct { func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResult, error) { const ( pageSize = 25 - baseURL = "https://www.reddit.com/" + baseURL = "https://www.reddit.com" maxRetries = 5 initialBackoff = 2 * time.Second ) var results []ForumSearchResult - searchURL := fmt.Sprintf("%ssearch.json?q=%s&limit=%d&start=%d", baseURL, url.QueryEscape(query), pageSize, page*pageSize) + searchURL := fmt.Sprintf("%s/search.json?q=%s&limit=%d&start=%d", baseURL, url.QueryEscape(query), pageSize, page*pageSize) var resp *http.Response var err error @@ -86,7 +85,7 @@ func PerformRedditSearch(query string, safe string, page int) ([]ForumSearchResu } publishedDate := time.Unix(int64(postData["created_utc"].(float64)), 0) permalink := postData["permalink"].(string) - resultURL := baseURL + permalink + resultURL := fmt.Sprintf("%s%s", baseURL, permalink) result := ForumSearchResult{ URL: resultURL, diff --git a/images.go b/images.go index a588348..572f8bc 100644 --- a/images.go +++ b/images.go @@ -51,7 +51,7 @@ func fetchImageResults(query string, safe, lang string, page int) ([]ImageSearch const resultsPerPage = 50 var offset int if page <= 1 { - offset = 0 // Assuming the API expects offset to start from 0 for the first page + offset = 0 } else { offset = (page - 1) * resultsPerPage } diff --git a/main.go b/main.go index 9a5ec3a..a10f225 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,3 @@ -// main.go package main import ( @@ -72,6 +71,7 @@ func main() { http.HandleFunc("/settings", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, "templates/settings.html") }) + initializeTorrentSites() fmt.Println("Server is listening on http://localhost:5000") log.Fatal(http.ListenAndServe(":5000", nil)) } @@ -88,11 +88,13 @@ func handleSearch(w http.ResponseWriter, r *http.Request) { case "image": handleImageSearch(w, query, safe, lang, page) case "video": - videoSearchEndpointHandler(w, r) + handleVideoSearch(w, query, safe, lang, page) case "map": handleMapSearch(w, query, safe) case "forum": handleForumsSearch(w, query, safe, lang, page) + case "file": + handleFileSearch(w, query, safe, lang, page) case "text": fallthrough default: diff --git a/run.sh b/run.sh index f56f9fc..c217373 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ #!/bin/bash -go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go --debug \ No newline at end of file +go run main.go text-google.go images.go imageproxy.go video.go map.go text.go text-quant.go text-duckduckgo.go cache.go forums.go files.go files-torrentgalaxy.go --debug \ No newline at end of file diff --git a/templates/files.html b/templates/files.html new file mode 100644 index 0000000..ebea712 --- /dev/null +++ b/templates/files.html @@ -0,0 +1,112 @@ + + + + + + {{.Query}} - Ocásek + + + +
+

Ocásek

+
+ + + +
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ + +
+
+
+ + +
+
+
+ + + + {{ if .Results }} +
+ + + + + +
+
+ {{ range .Results }} +
+ {{ if .Error }} +
{{ .Error }}
+ {{ else }} + {{ .URL }} +

{{ .Title }}

 +

{{ if .Views }}{{ .Views }} views • {{ end }}{{ .Size }}

+

Seeders: {{ .Seeders }} | Leechers: {{ .Leechers }}

+ {{ end }} +
+ {{ end }} + +
+
+
+ + + {{ if .HasPrevPage }} + + {{ end }} + {{ if .HasNextPage }} + + {{ end }} +
+
+ {{ else }} +
+ Your search '{{ .Query }}' came back with no results.
+ Try rephrasing your search term and/or recorrect any spelling mistakes. +
+ {{ end }} + + + diff --git a/templates/text.html b/templates/text.html index f056dc2..eef8e4c 100644 --- a/templates/text.html +++ b/templates/text.html @@ -38,8 +38,8 @@
- - + +
diff --git a/text-duckduckgo.go b/text-duckduckgo.go index 26ea84a..881256b 100644 --- a/text-duckduckgo.go +++ b/text-duckduckgo.go @@ -1,4 +1,3 @@ -// text-duckduckgo.go package main import ( diff --git a/text-google.go b/text-google.go index 2a9b53d..760128c 100644 --- a/text-google.go +++ b/text-google.go @@ -1,4 +1,3 @@ -// text-google.go package main import ( diff --git a/text-quant.go b/text-quant.go index c285100..c090ffe 100644 --- a/text-quant.go +++ b/text-quant.go @@ -1,4 +1,3 @@ -// text-qwant.go package main import ( diff --git a/text.go b/text.go index 0995e0e..c8342c5 100644 --- a/text.go +++ b/text.go @@ -173,22 +173,6 @@ func fetchAndCacheResults(query, safe, lang string, page, resultsPerPage int) [] return combinedResults } -func paginateResults(results []TextSearchResult, page, resultsPerPage int) []TextSearchResult { - startIndex := (page - 1) * resultsPerPage - endIndex := startIndex + resultsPerPage - - log.Printf("Paginating results: startIndex=%d, endIndex=%d, totalResults=%d", startIndex, endIndex, len(results)) - - if startIndex >= len(results) { - return []TextSearchResult{} - } - if endIndex > len(results) { - endIndex = len(results) - } - - return results[startIndex:endIndex] -} - func sourceOrder(source string) int { switch source { case "Google": diff --git a/video.go b/video.go index dc8c89b..766d8d5 100644 --- a/video.go +++ b/video.go @@ -7,7 +7,6 @@ import ( "log" "net/http" "net/url" - "strconv" "sync" "time" ) @@ -163,20 +162,6 @@ func makeHTMLRequest(query string) (*VideoAPIResponse, error) { return nil, fmt.Errorf("all instances failed, last error: %v", lastError) } -func videoSearchEndpointHandler(w http.ResponseWriter, r *http.Request) { - query := r.URL.Query().Get("q") - safe := r.URL.Query().Get("safe") // Assuming "safe" is a query parameter - lang := r.URL.Query().Get("lang") // Assuming "lang" is a query parameter - pageStr := r.URL.Query().Get("page") - page, err := strconv.Atoi(pageStr) - if err != nil { - // Handle error, perhaps set page to a default value if it's not critical - page = 1 // Default page - } - - handleVideoSearch(w, query, safe, lang, page) -} - // handleVideoSearch adapted from the Python `videoResults`, handles video search requests func handleVideoSearch(w http.ResponseWriter, query, safe, lang string, page int) { start := time.Now()