Add OpenAI-powered content parsing improvement feature

Author: paskal

backend/extractor/readability.go

@@ -394,6 +394,114 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
 	return rb, nil
 }
 
+// ContentParsedWrong handles the logic for when content is parsed incorrectly
+func (f *UReadability) ContentParsedWrong(ctx context.Context, urlStr string) (string, error) {
+	// Extract content using the current method
+	originalContent, err := f.Extract(ctx, urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract content: %v", err)
+	}
+
+	// Get CSS selector from ChatGPT
+	selector, err := f.getChatGPTSelector(ctx, urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to get CSS selector: %v", err)
+	}
+
+	// Get the HTML body
+	body, err := f.getHTMLBody(urlStr)
+	if err != nil {
+		return "", fmt.Errorf("failed to get HTML body: %v", err)
+	}
+
+	// Extract content using the new selector
+	newContent, err := f.extractContentWithSelector(body, selector)
+	if err != nil {
+		return "", fmt.Errorf("failed to extract content with new selector: %v", err)
+	}
+
+	// Compare original and new content
+	if strings.TrimSpace(originalContent.Content) != strings.TrimSpace(newContent) {
+		// Contents are different, create a new rule
+		rule := datastore.Rule{
+			Author:   "",
+			Domain:   f.extractDomain(urlStr),
+			Content:  selector,
+			TestURLs: []string{urlStr},
+			Enabled:  true,
+		}
+
+		_, err = f.Rules.Save(ctx, rule)
+		if err != nil {
+			return "", fmt.Errorf("failed to save new rule: %v", err)
+		}
+
+		return fmt.Sprintf("new custom rule with DOM %s created", selector), nil
+	}
+
+	return "default rule is good, no need to create the custom one", nil
+}
+
+func (f *UReadability) getChatGPTSelector(ctx context.Context, urlStr string) (string, error) {
+	client := openai.NewClient(f.OpenAIKey)
+	resp, err := client.CreateChatCompletion(
+		ctx,
+		openai.ChatCompletionRequest{
+			Model: openai.GPT4o,
+			Messages: []openai.ChatCompletionMessage{
+				{
+					Role:    openai.ChatMessageRoleSystem,
+					Content: "You are a helpful assistant that provides CSS selectors for extracting main content from web pages.",
+				},
+				{
+					Role:    openai.ChatMessageRoleUser,
+					Content: fmt.Sprintf("Given the URL %s, identify the CSS selector that can be used to extract the main content of the article. This typically includes elements like 'article', 'main', or specific classes. Return only this selector and nothing else.", urlStr),
+				},
+			},
+		},
+	)
+
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Choices[0].Message.Content, nil
+}
+
+func (f *UReadability) getHTMLBody(urlStr string) (string, error) {
+	//nolint:gosec
+	resp, err := http.Get(urlStr)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
+func (f *UReadability) extractContentWithSelector(body, selector string) (string, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(body))
+	if err != nil {
+		return "", err
+	}
+
+	content := doc.Find(selector).Text()
+	return content, nil
+}
+
+func (f *UReadability) extractDomain(urlStr string) string {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return ""
+	}
+	return u.Hostname()
+}
+
 // getContent retrieves content from raw body string, both content (text only) and rich (with html tags)
 // if rule is provided, it uses custom rule, otherwise tries to retrieve one from the storage,
 // and at last tries to use general readability parser

backend/rest/server.go

@@ -80,6 +80,7 @@ func (s *Server) routes(frontendDir string) http.Handler {
 			api.HandleFunc("POST /extract", s.extractArticle)
 			api.HandleFunc("POST /auth", s.authFake)
 			api.HandleFunc("GET /metrics", s.handleMetrics)
+			api.HandleFunc("GET /content-parsed-wrong", s.contentParsedWrong)
 
 			// add protected group with its own set of middlewares
 			protectedGroup := api.Group()
@@ -224,6 +225,15 @@ func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http
 		res.Summary = summaryText
 	}
 
+	if summary {
+		summaryText, err := s.Readability.GenerateSummary(r.Context(), res.Content)
+		if err != nil {
+			rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, fmt.Sprintf("failed to generate summary: %v", err))
+			return
+		}
+		res.Summary = summaryText
+	}
+
 	rest.RenderJSON(w, &res)
 }
 
@@ -409,6 +419,27 @@ func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) {
 	})
 }
 
+func (s *Server) contentParsedWrong(w http.ResponseWriter, r *http.Request) {
+	if s.Readability.OpenAIKey == "" {
+		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, nil, "OpenAI key is not set")
+		return
+	}
+
+	exampleURL := r.URL.Query().Get("url")
+	if exampleURL == "" {
+		rest.SendErrorJSON(w, r, log.Default(), http.StatusBadRequest, nil, "url parameter is required")
+		return
+	}
+
+	message, err := s.Readability.ContentParsedWrong(r.Context(), exampleURL)
+	if err != nil {
+		rest.SendErrorJSON(w, r, log.Default(), http.StatusInternalServerError, err, err.Error())
+		return
+	}
+
+	rest.RenderJSON(w, JSON{"message": message})
+}
+
 func getBid(id string) primitive.ObjectID {
 	bid, err := primitive.ObjectIDFromHex(id)
 	if err != nil {