From 6ed06b05b129d84fe1aaec1407576c6ba4e1a64a Mon Sep 17 00:00:00 2001 From: partisan Date: Thu, 12 Sep 2024 22:11:39 +0200 Subject: [PATCH] i need better way to cache images --- go.mod | 3 +++ go.sum | 8 ++++++++ imageproxy.go | 21 ++++++--------------- images-bing.go | 9 +++++++-- images-deviantart.go | 2 +- images-quant.go | 41 ++++++++++++++++++++++++++++++----------- images.go | 6 +++--- main.go | 2 +- 8 files changed, 59 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index f6755b1..a747de0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,9 @@ require ( ) require ( + github.com/chai2010/webp v1.1.1 // indirect + github.com/disintegration/imaging v1.6.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + golang.org/x/image v0.20.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect ) diff --git a/go.sum b/go.sum index 7fab23f..4120d74 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,16 @@ github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VP github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/chai2010/webp v1.1.1 h1:jTRmEccAJ4MGrhFOrPMpNGIJ/eybIgwKpcACsrTEapk= +github.com/chai2010/webp v1.1.1/go.mod h1:0XVwvZWdjjdxpUEIf7b9g9VkHFnInUSYujwqTLEuldU= github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732 h1:XYUCaZrW8ckGWlCRJKCSoh/iFwlpX316a8yY9IFEzv8= github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= github.com/chromedp/chromedp v0.9.5 h1:viASzruPJOiThk7c5bueOUY91jGLJVximoEMGoH93rg= github.com/chromedp/chromedp v0.9.5/go.mod h1:D4I2qONslauw/C7INoCir1BJkSwBYMyZgx8X276z3+Y= github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -31,6 +35,10 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/image v0.20.0 h1:7cVCUjQwfL18gyBJOmYvptfSHS8Fb3YUDtfLIZ7Nbpw= +golang.org/x/image v0.20.0/go.mod h1:0a88To4CYVBAHp5FXJm8o7QbUl37Vd85ply1vyD8auM= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= diff --git a/imageproxy.go b/imageproxy.go index 74a1cbb..4dd7478 100644 --- a/imageproxy.go +++ b/imageproxy.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "io" "net/http" ) @@ -14,20 +13,12 @@ func handleImageProxy(w http.ResponseWriter, r *http.Request) { return } - // Try to fetch the image from Bing first - bingURL := fmt.Sprintf("https://tse.mm.bing.net/th?q=%s", imageURL) - resp, err := http.Get(bingURL) - if err != nil || resp.StatusCode != http.StatusOK { - // If fetching from Bing fails, attempt to fetch from the original image URL - printWarn("Error fetching image from Bing, trying original URL.") - - // Attempt to fetch the image directly - resp, err = http.Get(imageURL) - if err != nil { - printWarn("Error fetching image: %v", err) - http.Error(w, "Internal Server Error", http.StatusInternalServerError) - return - } + // Fetch the image from the external URL + resp, err := http.Get(imageURL) + if err != nil { + printWarn("Error fetching image: %v", err) + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + return } defer resp.Body.Close() diff --git a/images-bing.go b/images-bing.go index d6e37c5..48ba32d 100644 --- a/images-bing.go +++ b/images-bing.go @@ -12,6 +12,7 @@ import ( "github.com/PuerkitoBio/goquery" ) +// PerformBingImageSearch performs a Bing image search and returns the results. func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchResult, time.Duration, error) { startTime := time.Now() @@ -66,14 +67,16 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe if err := json.Unmarshal([]byte(metadata), &data); err == nil { mediaURL, ok := data["murl"].(string) if ok { + // Apply the image proxy + proxiedURL := "/imgproxy?url=" + mediaURL results = append(results, ImageSearchResult{ Thumbnail: imgSrc, Title: strings.TrimSpace(title), Media: mediaURL, + Source: mediaURL, + ThumbProxy: proxiedURL, // Use the proxied URL Width: width, Height: height, - Source: mediaURL, - ThumbProxy: imgSrc, }) } } @@ -89,6 +92,7 @@ func PerformBingImageSearch(query, safe, lang string, page int) ([]ImageSearchRe return results, duration, nil } +// buildBingSearchURL constructs the search URL for Bing Image Search func buildBingSearchURL(query string, page int) string { baseURL := "https://www.bing.com/images/search" params := url.Values{} @@ -99,6 +103,7 @@ func buildBingSearchURL(query string, page int) string { return baseURL + "?" + params.Encode() } +// Example usage in main (commented out for clarity) // func main() { // results, duration, err := PerformBingImageSearch("kittens", "false", "en", 1) // if err != nil { diff --git a/images-deviantart.go b/images-deviantart.go index cbeeea4..3b12def 100644 --- a/images-deviantart.go +++ b/images-deviantart.go @@ -156,7 +156,7 @@ func PerformDeviantArtImageSearch(query, safe, lang string, page int) ([]ImageSe Width: 0, Height: 0, Source: resultURL, - ThumbProxy: imgSrc, + ThumbProxy: "/imgproxy?url=" + imgSrc, } } }(imgSrc, resultURL, title) diff --git a/images-quant.go b/images-quant.go index 0dccdec..8690239 100644 --- a/images-quant.go +++ b/images-quant.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "sync" "time" ) @@ -141,19 +142,37 @@ func PerformQwantImageSearch(query, safe, lang string, page int) ([]ImageSearchR return nil, 0, fmt.Errorf("decoding response: %v", err) } - var results []ImageSearchResult - for _, item := range apiResp.Data.Result.Items { - results = append(results, ImageSearchResult{ - Thumbnail: item.Thumbnail, - Title: item.Title, - Media: item.Media, - Source: item.Url, - ThumbProxy: "/img_proxy?url=" + url.QueryEscape(item.Media), // New proxy not exactly working as intended - Width: item.Width, - Height: item.Height, - }) + var wg sync.WaitGroup + results := make([]ImageSearchResult, len(apiResp.Data.Result.Items)) + + for i, item := range apiResp.Data.Result.Items { + wg.Add(1) + go func(i int, item struct { + Media string `json:"media"` + Thumbnail string `json:"thumbnail"` + Title string `json:"title"` + Url string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + }) { + defer wg.Done() + + // Populate the result + results[i] = ImageSearchResult{ + Thumbnail: item.Thumbnail, + Title: item.Title, + Media: item.Media, + Source: item.Url, + ThumbProxy: "/imgproxy?url=" + item.Media, + Width: item.Width, + Height: item.Height, + } + }(i, item) } + // Wait for all goroutines to complete + wg.Wait() + duration := time.Since(startTime) // Calculate the duration return results, duration, nil diff --git a/images.go b/images.go index 4c6b957..5a202c7 100755 --- a/images.go +++ b/images.go @@ -13,9 +13,9 @@ var imageSearchEngines []SearchEngine func init() { imageSearchEngines = []SearchEngine{ {Name: "Qwant", Func: wrapImageSearchFunc(PerformQwantImageSearch), Weight: 1}, - {Name: "DeviantArt", Func: wrapImageSearchFunc(PerformDeviantArtImageSearch), Weight: 2}, - {Name: "Bing", Func: wrapImageSearchFunc(PerformBingImageSearch), Weight: 2}, // Bing sometimes returns with low amount of images, this leads to danamica page loading not working - {Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 3}, + {Name: "Bing", Func: wrapImageSearchFunc(PerformBingImageSearch), Weight: 2}, + {Name: "DeviantArt", Func: wrapImageSearchFunc(PerformDeviantArtImageSearch), Weight: 3}, + //{Name: "Imgur", Func: wrapImageSearchFunc(PerformImgurImageSearch), Weight: 4}, // Image proxy not working } } diff --git a/main.go b/main.go index 63b60ca..930b583 100755 --- a/main.go +++ b/main.go @@ -183,7 +183,7 @@ func runServer() { http.HandleFunc("/", handleSearch) http.HandleFunc("/search", handleSearch) http.HandleFunc("/suggestions", handleSuggestions) - http.HandleFunc("/img_proxy", handleImageProxy) + http.HandleFunc("/imgproxy", handleImageProxy) http.HandleFunc("/node", handleNodeRequest) http.HandleFunc("/settings", handleSettings) http.HandleFunc("/save-settings", handleSaveSettings)