feat(datastore): add SQLite database backend support

.github/workflows/check-db-schema-structs.yaml

@@ -51,3 +51,23 @@ jobs:
               git diff
               exit 1
           fi
+  check-sqlite-schema-structs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.24.4"
+      - name: Generate SQLite DB schema structs
+        run: make gen/gorm/sqlite
+      - name: Check if there are uncommitted file changes
+        run: |
+          clean=$(git status --porcelain)
+          if [[ -z "$clean" ]]; then
+              echo "SQLite schema is up to date."
+          else
+              echo "Uncommitted file changes detected after generating SQLite schema: $clean"
+              git diff
+              exit 1
+          fi

Makefile

@@ -114,6 +114,16 @@ start/postgres:
 stop/postgres:
 	./scripts/teardown_postgres_db.sh
 
+# Start the SQLite database (file-based, minimal setup)
+.PHONY: start/sqlite
+start/sqlite:
+	./scripts/start_sqlite_db.sh
+
+# Stop the SQLite database (cleanup)
+.PHONY: stop/sqlite
+stop/sqlite:
+	./scripts/teardown_sqlite_db.sh
+
 # generate the gorm structs for MySQL
 .PHONY: gen/gorm/mysql
 gen/gorm/mysql: bin/golang-migrate start/mysql
@@ -129,12 +139,22 @@ gen/gorm/postgres: bin/golang-migrate start/postgres
 	cd gorm-gen && GOWORK=off go run main.go --db-type postgres --dsn 'postgres://postgres:postgres@localhost:5432/model-registry?sslmode=disable' && \
 	cd $(CURDIR) && ./scripts/remove_gorm_defaults.sh)
 
+# generate the gorm structs for SQLite
+.PHONY: gen/gorm/sqlite
+gen/gorm/sqlite: bin/golang-migrate start/sqlite
+	@(trap 'cd $(CURDIR) && $(MAKE) stop/sqlite && rm -f /tmp/gorm-gen-sqlite.db' EXIT; \
+	$(GOLANG_MIGRATE) -path './internal/datastore/embedmd/sqlite/migrations' -database 'sqlite:///tmp/gorm-gen-sqlite.db' up && \
+	cd gorm-gen && GOWORK=off go run main.go --db-type sqlite --dsn '/tmp/gorm-gen-sqlite.db')
+
 # generate the gorm structs (defaults to MySQL for backward compatibility)
 # Use GORM_DB_TYPE=postgres to generate for PostgreSQL instead
+# Use GORM_DB_TYPE=sqlite to generate for SQLite instead
 .PHONY: gen/gorm
 gen/gorm: bin/golang-migrate
 ifeq ($(GORM_DB_TYPE),postgres)
 	$(MAKE) gen/gorm/postgres
+else ifeq ($(GORM_DB_TYPE),sqlite)
+	$(MAKE) gen/gorm/sqlite
 else
 	$(MAKE) gen/gorm/mysql
 endif
@@ -185,7 +205,7 @@ bin/yq:
 
 GOLANG_MIGRATE ?= ${PROJECT_BIN}/migrate
 bin/golang-migrate:
-	GOBIN=$(PROJECT_PATH)/bin ${GO} install -tags 'mysql,postgres' github.com/golang-migrate/migrate/v4/cmd/[email protected]
+	GOBIN=$(PROJECT_PATH)/bin ${GO} install -tags 'mysql,postgres,sqlite' github.com/golang-migrate/migrate/v4/cmd/[email protected]
 
 GENQLIENT ?= ${PROJECT_BIN}/genqlient
 bin/genqlient:

go.mod

@@ -28,6 +28,7 @@ require (
 	google.golang.org/protobuf v1.36.8
 	gorm.io/driver/mysql v1.6.0
 	gorm.io/driver/postgres v1.6.0
+	gorm.io/driver/sqlite v1.6.0
 	gorm.io/gorm v1.30.1
 	k8s.io/api v0.33.4
 	k8s.io/apimachinery v0.33.4
@@ -102,6 +103,7 @@ require (
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/lib/pq v1.10.9 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
 	github.com/moby/go-archive v0.1.0 // indirect
 	github.com/moby/sys/user v0.4.0 // indirect

go.sum

@@ -303,6 +303,8 @@ github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8S
 github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
 github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
 github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
 github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
@@ -661,6 +663,8 @@ gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
 gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
 gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
 gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
+gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
+gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
 gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
 gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=

go.work.sum

@@ -662,6 +662,8 @@ gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKK
 gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
 gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
+gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
+gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc h1:/hemPrYIhOhy8zYrNj+069zDB68us2sMGsfkFJO0iZs=
 istio.io/api v1.24.2 h1:jYjcN6Iq0RPtQj/3KMFsybxmfqmjGN/dxhL7FGJEdIM=
 istio.io/api v1.24.2/go.mod h1:MQnRok7RZ20/PE56v0LxmoWH0xVxnCQPNuf9O7PAN1I=

gorm-gen/main.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"strings"
 
 	"github.com/spf13/cobra"
 	"gorm.io/driver/mysql"
@@ -30,31 +29,135 @@ func genModels(g *gen.Generator, db *gorm.DB, tables []string) (err error) {
 		}
 	}
 
-	// Custom ModelOpt to remove default tag for nullable fields
+	// Custom ModelOpt to clean up primary key tags
 	modelOpt := gen.FieldGORMTag("*", func(tag field.GormTag) field.GormTag {
-		if vals, ok := tag["default"]; ok {
-			if len(vals) > 0 {
-				val := strings.Trim(strings.TrimSpace(vals[0]), `"'`)
-				if strings.ToUpper(val) == "NULL" || val == "0" || val == "" {
-					tag.Remove("default")
-				}
-			}
+		// Remove "not null" from primary key fields since it's redundant
+		if _, hasPrimaryKey := tag["primaryKey"]; hasPrimaryKey {
+			tag.Remove("not null")
 		}
 		return tag
 	})
 
 	// Execute some data table tasks
 	for _, tableName := range tables {
 		if tableName == "Type" {
-			// Special handling for Type table to set TypeKind as int32
-			g.GenerateModel(tableName, gen.FieldType("type_kind", "int32"), modelOpt)
+			// Special handling for Type table
+			if dbType == "sqlite" {
+				g.GenerateModel(tableName, 
+					gen.FieldType("type_kind", "int32"), 
+					gen.FieldType("id", "int32"),
+					gen.FieldGORMTag("id", func(tag field.GormTag) field.GormTag {
+						tag.Set("column", "id")
+						tag.Set("primaryKey", "")
+						tag.Set("autoIncrement", "true")
+						return tag
+					}),
+					modelOpt)
+			} else {
+				g.GenerateModel(tableName, 
+					gen.FieldType("type_kind", "int32"), 
+					gen.FieldGORMTag("id", func(tag field.GormTag) field.GormTag {
+						tag.Set("column", "id")
+						tag.Set("primaryKey", "")
+						tag.Set("autoIncrement", "true")
+						return tag
+					}),
+					modelOpt)
+			}
+		} else if tableName == "schema_migrations" && dbType == "sqlite" {
+			// Special handling for schema_migrations table in SQLite to match MySQL/PostgreSQL types
+			g.GenerateModel(tableName, 
+				gen.FieldType("version", "int64"),
+				gen.FieldType("dirty", "bool"), 
+				gen.FieldGORMTag("version", func(tag field.GormTag) field.GormTag {
+					tag.Set("column", "version")
+					tag.Set("primaryKey", "")
+					return tag
+				}),
+				gen.FieldGORMTag("dirty", func(tag field.GormTag) field.GormTag {
+					tag.Set("column", "dirty")
+					tag.Set("not null", "")
+					return tag
+				}),
+				modelOpt)
+		} else if isPropertyTable(tableName) {
+			// Special handling for property tables to ensure composite primary keys
+			var opts []gen.ModelOpt
+			if dbType == "sqlite" && hasIDField(tableName) {
+				opts = append(opts, gen.FieldType("id", "int32"))
+				opts = append(opts, gen.FieldGORMTag("id", func(tag field.GormTag) field.GormTag {
+					tag.Set("column", "id")
+					tag.Set("primaryKey", "")
+					tag.Set("autoIncrement", "true")
+					return tag
+				}))
+			}
+
+			// Fix composite primary key fields
+			opts = append(opts, gen.FieldGORMTag("name", func(tag field.GormTag) field.GormTag {
+				tag.Set("column", "name")
+				tag.Set("primaryKey", "")
+				tag.Remove("not null") // Remove not null since primaryKey implies it
+				return tag
+			}))
+			opts = append(opts, gen.FieldGORMTag("is_custom_property", func(tag field.GormTag) field.GormTag {
+				tag.Set("column", "is_custom_property")
+				tag.Set("primaryKey", "")
+				tag.Remove("not null") // Remove not null since primaryKey implies it
+				return tag
+			}))
+			opts = append(opts, modelOpt)
+			g.GenerateModel(tableName, opts...)
+		} else if hasIDField(tableName) {
+			// Tables with ID fields - ensure autoIncrement is set
+			var opts []gen.ModelOpt
+			if dbType == "sqlite" {
+				opts = append(opts, gen.FieldType("id", "int32"))
+			}
+			opts = append(opts, gen.FieldGORMTag("id", func(tag field.GormTag) field.GormTag {
+				tag.Set("column", "id")
+				tag.Set("primaryKey", "")
+				tag.Set("autoIncrement", "true")
+				return tag
+			}))
+			opts = append(opts, modelOpt)
+			g.GenerateModel(tableName, opts...)
 		} else {
 			g.GenerateModel(tableName, modelOpt)
 		}
 	}
 	return nil
 }
 
+// hasIDField returns true if the table has an auto-increment ID primary key field
+func hasIDField(tableName string) bool {
+	tablesWithIDField := []string{
+		"Artifact", "Association", "Attribution", "Context", 
+		"Event", "Execution", "Type",
+	}
+
+	for _, table := range tablesWithIDField {
+		if table == tableName {
+			return true
+		}
+	}
+	return false
+}
+
+// isPropertyTable returns true if the table is a property table with composite primary keys
+func isPropertyTable(tableName string) bool {
+	propertyTables := []string{
+		"ArtifactProperty", "ContextProperty", "ExecutionProperty", "TypeProperty",
+	}
+
+	for _, table := range propertyTables {
+		if table == tableName {
+			return true
+		}
+	}
+	return false
+}
+
 // getDialector returns the appropriate GORM dialector based on database type and DSN
 func getDialector(dbType, dsn string) (gorm.Dialector, error) {
 	switch dbType {

internal/datastore/embedmd/service.go

@@ -20,8 +20,8 @@ type EmbedMDConfig struct {
 }
 
 func (c *EmbedMDConfig) Validate() error {
-	if c.DatabaseType != types.DatabaseTypeMySQL && c.DatabaseType != types.DatabaseTypePostgres {
-		return fmt.Errorf("unsupported database type: %s. Supported types: %s, %s", c.DatabaseType, types.DatabaseTypeMySQL, types.DatabaseTypePostgres)
+	if c.DatabaseType != types.DatabaseTypeMySQL && c.DatabaseType != types.DatabaseTypePostgres && c.DatabaseType != types.DatabaseTypeSQLite {
+		return fmt.Errorf("unsupported database type: %s. Supported types: %s, %s, %s", c.DatabaseType, types.DatabaseTypeMySQL, types.DatabaseTypePostgres, types.DatabaseTypeSQLite)
 	}
 
 	return nil

internal/datastore/embedmd/sqlite/connect.go

@@ -0,0 +1,99 @@
+package sqlite
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/golang/glog"
+	_tls "github.com/kubeflow/model-registry/internal/tls"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+const (
+	// sqliteMaxRetriesDefault is the maximum number of attempts to retry SQLite connection.
+	sqliteMaxRetriesDefault = 5 // SQLite is file-based, so fewer retries needed
+)
+
+type SQLiteDBConnector struct {
+	DSN          string
+	TLSConfig    *_tls.TLSConfig // Not used for SQLite but kept for interface consistency
+	db           *gorm.DB
+	connectMutex sync.Mutex
+	maxRetries   int
+}
+
+func NewSQLiteDBConnector(
+	dsn string,
+	tlsConfig *_tls.TLSConfig,
+) *SQLiteDBConnector {
+	return &SQLiteDBConnector{
+		DSN:        dsn,
+		TLSConfig:  tlsConfig,
+		maxRetries: sqliteMaxRetriesDefault,
+	}
+}
+
+func (c *SQLiteDBConnector) WithMaxRetries(maxRetries int) *SQLiteDBConnector {
+	c.maxRetries = maxRetries
+
+	return c
+}
+
+func (c *SQLiteDBConnector) Connect() (*gorm.DB, error) {
+	// Use mutex to ensure only one connection attempt at a time
+	c.connectMutex.Lock()
+	defer c.connectMutex.Unlock()
+
+	// If we already have a working connection, return it
+	if c.db != nil {
+		return c.db, nil
+	}
+
+	var db *gorm.DB
+	var err error
+
+	// Log warning if TLS configuration is specified (not supported by SQLite)
+	if c.TLSConfig != nil && c.needsTLSConfig() {
+		glog.Warningf("TLS configuration is not supported for SQLite connections, ignoring TLS settings")
+	}
+
+	for i := range c.maxRetries {
+		glog.V(2).Infof("Attempting to connect to SQLite database: %q (attempt %d/%d)", c.DSN, i+1, c.maxRetries)
+		db, err = gorm.Open(sqlite.Open(c.DSN), &gorm.Config{
+			Logger:         logger.Default.LogMode(logger.Silent),
+			TranslateError: true,
+		})
+		if err == nil {
+			break
+		}
+
+		glog.Warningf("Retrying connection to SQLite (attempt %d/%d): %v", i+1, c.maxRetries, err)
+
+		time.Sleep(time.Duration(i+1) * time.Second)
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to connect to SQLite: %w", err)
+	}
+
+	glog.Info("Successfully connected to SQLite database")
+
+	c.db = db
+
+	return db, nil
+}
+
+func (c *SQLiteDBConnector) DB() *gorm.DB {
+	return c.db
+}
+
+func (c *SQLiteDBConnector) needsTLSConfig() bool {
+	if c.TLSConfig == nil {
+		return false
+	}
+
+	return c.TLSConfig.CertPath != "" || c.TLSConfig.KeyPath != "" || c.TLSConfig.RootCertPath != "" || c.TLSConfig.CAPath != "" || c.TLSConfig.Cipher != "" || c.TLSConfig.VerifyServerCert
+}