Add article summary feature with OpenAI integration

Implement MongoDB cache for summaries to reduce API costs and improve performance.
Rename parameters from OpenAI-specific to more generic API names.
Support configurable model selection through ModelType enum or direct model names.
Add comprehensive tests with mocks for summary generation and caching.
Update documentation in README with summary feature details.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: paskal

README.md

@@ -11,7 +11,8 @@
 | address      | UKEEPER_ADDRESS | all interfaces | web server listening address                          |
 | port         | UKEEPER_PORT    | `8080`         | web server port                                       |
 | mongo_uri    | MONGO_URI       | none           | MongoDB connection string, _required_                 |
-| openai_key   | OPENAI_KEY      | none           | OpenAI API key for summary generation                 |
+| api_key      | API_KEY         | none           | OpenAI API key for summary generation                 |
+| model_type   | MODEL_TYPE      | `mini`         | OpenAI model for summary generation (mini, default, large, or direct model name like gpt-4o) |
 | frontend_dir | FRONTEND_DIR    | `/srv/web`     | directory with frontend files                         |
 | token        | TOKEN           | none           | token for /content/v1/parser endpoint auth            |
 | mongo-delay  | MONGO_DELAY     | `0`            | mongo initial delay                                   |
@@ -24,6 +25,31 @@
     GET /api/content/v1/parser?token=secret&summary=true&url=http://aa.com/blah - extract content (emulate Readability API parse call), summary is optional and requires OpenAI key and token to be enabled
     POST /api/v1/extract {url: http://aa.com/blah}  - extract content
 
+### Article Summary Feature
+
+The application can generate concise summaries of article content using OpenAI's GPT models:
+
+1. **Configuration**:
+   - Set `openai_key` to your OpenAI API key
+   - Optionally set `openai_model` to specify which model to use:
+     - Simplified options: `mini` (default, uses gpt-4o-mini), `default` (uses gpt-4o), `large` (uses gpt-4o)
+     - Direct specification: Any valid OpenAI model name (e.g., `gpt-4o`, `gpt-4o-mini`, etc.)
+   - A server token must be configured for security reasons
+
+2. **Usage**:
+   - Add `summary=true` parameter to the `/api/content/v1/parser` endpoint
+   - Example: `/api/content/v1/parser?token=secret&summary=true&url=http://example.com/article`
+
+3. **Features**:
+   - Summaries are cached in MongoDB to reduce API costs and improve performance
+   - The cache stores:
+     - Content hash (to identify articles)
+     - Summary text
+     - Model used for generation
+     - Creation and update timestamps
+   - If the same content is requested again, the cached summary is returned
+   - The preview page automatically shows summaries when available
+
 ## Development
 
 ### Running tests

backend/datastore/mongo.go

@@ -37,15 +37,29 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error
 	return &MongoServer{client: client, dbName: dbName}, nil
 }
 
+// Stores contains all DAO instances
+type Stores struct {
+	Rules     RulesDAO
+	Summaries SummariesDAO
+}
+
 // GetStores initialize collections and make indexes
-func (m *MongoServer) GetStores() (rules RulesDAO) {
+func (m *MongoServer) GetStores() Stores {
 	rIndexes := []mongo.IndexModel{
 		{Keys: bson.D{{Key: "enabled", Value: 1}, {Key: "domain", Value: 1}}},
 		{Keys: bson.D{{Key: "user", Value: 1}, {Key: "domain", Value: 1}, {Key: "enabled", Value: 1}}},
 		{Keys: bson.D{{Key: "domain", Value: 1}, {Key: "match_urls", Value: 1}}},
 	}
-	rules = RulesDAO{Collection: m.collection("rules", rIndexes)}
-	return rules
+
+	sIndexes := []mongo.IndexModel{
+		{Keys: bson.D{{Key: "created_at", Value: 1}}},
+		{Keys: bson.D{{Key: "model", Value: 1}}},
+	}
+
+	return Stores{
+		Rules:     RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Summaries: SummariesDAO{Collection: m.collection("summaries", sIndexes)},
+	}
 }
 
 // collection makes collection with indexes

backend/datastore/rules_test.go

@@ -19,7 +19,9 @@ func TestRules(t *testing.T) {
 	server, err := New("mongodb://localhost:27017/", "test_ureadability", 0)
 	require.NoError(t, err)
 	assert.NotNil(t, server.client)
-	rules := server.GetStores()
+	stores := server.GetStores()
+	assert.NotNil(t, stores)
+	rules := stores.Rules
 	assert.NotNil(t, rules)
 	rule := Rule{
 		Domain:  randStringBytesRmndr(42) + ".com",
@@ -74,7 +76,9 @@ func TestRulesCanceledContext(t *testing.T) {
 	server, err := New("mongodb://wrong", "", 0)
 	require.NoError(t, err)
 	assert.NotNil(t, server.client)
-	rules := server.GetStores()
+	stores := server.GetStores()
+	assert.NotNil(t, stores)
+	rules := stores.Rules
 	assert.NotNil(t, rules)
 
 	ctx, cancel := context.WithCancel(context.Background())

backend/datastore/summaries.go

@@ -0,0 +1,90 @@
+// Package datastore provides mongo implementation for store to keep and access summaries
+package datastore
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+// Summary contains information about a cached summary
+type Summary struct {
+	ID        string    `bson:"_id"`     // SHA256 hash of the content
+	Content   string    `bson:"content"` // Original content that was summarized (could be truncated for storage efficiency)
+	Summary   string    `bson:"summary"` // Generated summary
+	Model     string    `bson:"model"`   // OpenAI model used for summarization
+	CreatedAt time.Time `bson:"created_at"`
+	UpdatedAt time.Time `bson:"updated_at"`
+}
+
+// SummariesDAO handles database operations for article summaries
+type SummariesDAO struct {
+	Collection *mongo.Collection
+}
+
+// Get returns summary by content hash
+func (s SummariesDAO) Get(ctx context.Context, content string) (Summary, bool) {
+	contentHash := generateContentHash(content)
+	res := s.Collection.FindOne(ctx, bson.M{"_id": contentHash})
+	if res.Err() != nil {
+		if res.Err() == mongo.ErrNoDocuments {
+			return Summary{}, false
+		}
+		log.Printf("[WARN] can't get summary for hash %s: %v", contentHash, res.Err())
+		return Summary{}, false
+	}
+
+	summary := Summary{}
+	if err := res.Decode(&summary); err != nil {
+		log.Printf("[WARN] can't decode summary document for hash %s: %v", contentHash, err)
+		return Summary{}, false
+	}
+
+	return summary, true
+}
+
+// Save creates or updates summary in the database
+func (s SummariesDAO) Save(ctx context.Context, summary Summary) error {
+	if summary.ID == "" {
+		summary.ID = generateContentHash(summary.Content)
+	}
+
+	if summary.CreatedAt.IsZero() {
+		summary.CreatedAt = time.Now()
+	}
+	summary.UpdatedAt = time.Now()
+
+	opts := options.Update().SetUpsert(true)
+	_, err := s.Collection.UpdateOne(
+		ctx,
+		bson.M{"_id": summary.ID},
+		bson.M{"$set": summary},
+		opts,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to save summary: %w", err)
+	}
+	return nil
+}
+
+// Delete removes summary from the database
+func (s SummariesDAO) Delete(ctx context.Context, contentHash string) error {
+	_, err := s.Collection.DeleteOne(ctx, bson.M{"_id": contentHash})
+	if err != nil {
+		return fmt.Errorf("failed to delete summary: %w", err)
+	}
+	return nil
+}
+
+// generateContentHash creates a hash for the content to use as an ID
+func generateContentHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}

backend/datastore/summaries_test.go

@@ -0,0 +1,88 @@
+package datastore
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSummariesDAO_SaveAndGet(t *testing.T) {
+	if _, ok := os.LookupEnv("ENABLE_MONGO_TESTS"); !ok {
+		t.Skip("ENABLE_MONGO_TESTS env variable is not set")
+	}
+
+	mdb, err := New("mongodb://localhost:27017", "test_ureadability", 0)
+	require.NoError(t, err)
+
+	// Create a unique collection for this test to avoid conflicts
+	collection := mdb.client.Database(mdb.dbName).Collection("summaries_test")
+	defer func() {
+		_ = collection.Drop(context.Background())
+	}()
+
+	dao := SummariesDAO{Collection: collection}
+
+	content := "This is a test article content. It should generate a unique hash."
+	summary := Summary{
+		Content:   content,
+		Summary:   "This is a test summary of the article.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: time.Now(),
+	}
+
+	// Test saving a summary
+	err = dao.Save(context.Background(), summary)
+	require.NoError(t, err)
+
+	// Test getting the summary
+	foundSummary, found := dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, summary.Summary, foundSummary.Summary)
+	assert.Equal(t, summary.Model, foundSummary.Model)
+	assert.NotEmpty(t, foundSummary.ID)
+
+	// Test getting a non-existent summary
+	_, found = dao.Get(context.Background(), "non-existent content")
+	assert.False(t, found)
+
+	// Test updating an existing summary
+	updatedSummary := Summary{
+		ID:        foundSummary.ID,
+		Content:   content,
+		Summary:   "This is an updated summary.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: foundSummary.CreatedAt,
+	}
+
+	err = dao.Save(context.Background(), updatedSummary)
+	require.NoError(t, err)
+
+	foundSummary, found = dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, "This is an updated summary.", foundSummary.Summary)
+	assert.Equal(t, updatedSummary.CreatedAt, foundSummary.CreatedAt)
+	assert.NotEqual(t, updatedSummary.UpdatedAt, foundSummary.UpdatedAt) // UpdatedAt should be set by the DAO
+
+	// Test deleting a summary
+	err = dao.Delete(context.Background(), foundSummary.ID)
+	require.NoError(t, err)
+
+	_, found = dao.Get(context.Background(), content)
+	assert.False(t, found)
+}
+
+func TestGenerateContentHash(t *testing.T) {
+	content1 := "This is a test content."
+	content2 := "This is a different test content."
+
+	hash1 := generateContentHash(content1)
+	hash2 := generateContentHash(content2)
+
+	assert.NotEqual(t, hash1, hash2)
+	assert.Equal(t, hash1, generateContentHash(content1)) // Same content should produce same hash
+	assert.Equal(t, 64, len(hash1))                       // SHA-256 produces 64 character hex string
+}

backend/extractor/pics_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/stretchr/testify/assert"
@@ -27,7 +28,7 @@ func TestExtractPics(t *testing.T) {
 	defer ts.Close()
 
 	t.Log("test main pic")
-	lr := UReadability{TimeOut: 30, SnippetSize: 200}
+	lr := UReadability{TimeOut: time.Second * 30, SnippetSize: 200}
 	a, err := lr.Extract(context.Background(), ts.URL+"/2015/09/25/poiezdka-s-apple-maps/")
 	require.NoError(t, err)
 	allImages := []string{
@@ -41,7 +42,7 @@ func TestExtractPics(t *testing.T) {
 
 func TestExtractPicsDirectly(t *testing.T) {
 	t.Log("test pic directly")
-	lr := UReadability{TimeOut: 30, SnippetSize: 200}
+	lr := UReadability{TimeOut: 30 * time.Second, SnippetSize: 200}
 	t.Run("normal image retrieval", func(t *testing.T) {
 		data := `<body>
 		<img class="alignright size-full wp-image-944214 lazyloadableImage lazyLoad-fadeIn" alt="View Page Source" width="308" height="508" data-original="https://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg" src="https://cdn1.tnwcdn.com/wp-content/blogs.dir/1/files/2016/01/page-source.jpg"></body>`