Schema cleaning.

Author: fengttt

t2sql/t2sql/common/common.go

@@ -35,9 +35,16 @@ func ProjectRoot() string {
 	return root
 }
 
+type ColInfo struct {
+	Name        string
+	Type        string
+	Description string
+}
+
 type TableInfo struct {
-	Name string // table name
-	Sql  string // create table sql statement
+	Name     string // table name
+	Sql      string // create table sql statement
+	ColInfos []ColInfo
 }
 
 type DbInfo struct {

t2sql/t2sql/common/spider2.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-type ColInfo struct {
+type ColInfoParsed struct {
 	TableName   string   `json:"table_name"`
 	Colnames    []string `json:"column_names"`
 	Coltypes    []string `json:"column_types"`
@@ -38,34 +38,71 @@ func loadOneDbInfo(dbInfoDir string, sqliteDir string, dbName string) (*DbInfo,
 			// replace NaN with null, NaN is not a valid json value
 			jsonData = []byte(strings.ReplaceAll(string(jsonData), ": NaN", ": null"))
 
-			var colInfo ColInfo
-			if err := json.Unmarshal(jsonData, &colInfo); err != nil {
+			var colParse ColInfoParsed
+			if err := json.Unmarshal(jsonData, &colParse); err != nil {
 				return nil, fmt.Errorf("cannot unmarshal json file: %s, %w", jsonFile, err)
 			}
 
 			var tableInfo TableInfo
-			tableInfo.Name = colInfo.TableName
-			tableInfo.Sql = fmt.Sprintf("CREATE TABLE %s (", tableInfo.Name)
+			tableInfo.Name = colParse.TableName
+			switch strings.ToLower(tableInfo.Name) {
+			case "match":
+				tableInfo.Name = "match_table"
+			}
 
-			for i, colname := range colInfo.Colnames {
-				coltype := colInfo.Coltypes[i]
+			tableInfo.Sql = fmt.Sprintf("CREATE TABLE %s (", tableInfo.Name)
+			for i, colname := range colParse.Colnames {
+				coltype := colParse.Coltypes[i]
 				switch colname {
-				case "rank":
-					colname = fmt.Sprintf("%s_rank", tableInfo.Name)
+				case "rank", "index", "table", "column", "group", "range",
+					"cross", "change":
+					colname = fmt.Sprintf("%s_%s", tableInfo.Name, colname)
 				}
-				if coltype == "" {
-					coltype = "text"
+
+				if strings.HasPrefix(colname, "Unnamed: ") {
+					colname = fmt.Sprintf("unnamed_%s", colname[len("Unnamed: "):])
+				}
+
+				colname = strings.ReplaceAll(colname, "%", "percent")
+				colname = strings.ReplaceAll(colname, "/", "_over_")
+				colname = strings.ReplaceAll(colname, "(", "_")
+				colname = strings.ReplaceAll(colname, ")", "_")
+				colname = strings.ReplaceAll(colname, "-", "_")
+
+				if strings.HasSuffix(colname, "%") {
+					colname = fmt.Sprintf("%s_percent", colname[:len(colname)-1])
 				}
-				if coltype == "NUM" {
+
+				switch coltype {
+				case "NUM":
 					coltype = "float"
+				case "", "BLOB SUB_TYPE TEXT", "point":
+					coltype = "text"
+				case "jsonb":
+					coltype = "json"
+				case "timestamp with time zone":
+					coltype = "timestamp"
+				}
+
+				if strings.HasPrefix(strings.ToLower(coltype), "nvarchar") {
+					coltype = "text"
+				}
+
+				if strings.HasPrefix(strings.ToLower(coltype), "character") {
+					coltype = "varchar(255)"
 				}
 
 				tableInfo.Sql += fmt.Sprintf("%s %s", colname, coltype)
-				if i == len(colInfo.Colnames)-1 {
+				if i == len(colParse.Colnames)-1 {
 					tableInfo.Sql += ");\n"
 				} else {
 					tableInfo.Sql += ",\n"
 				}
+				tableInfo.ColInfos = append(tableInfo.ColInfos, ColInfo{
+					Name:        colname,
+					Type:        coltype,
+					Description: colParse.Description[i],
+				})
 			}
 			dbInfo.TableInfos = append(dbInfo.TableInfos, tableInfo)
 		}
@@ -94,3 +131,24 @@ func Spider2LoadDbInfo() (map[string]*DbInfo, error) {
 
 	return dbInfos, nil
 }
+
+func Spider2CreateMoTables(dbInfo *DbInfo) error {
+	dbName := dbInfo.Name
+	mo, err := OpenMoDB()
+	if err != nil {
+		return fmt.Errorf("failed to open mo db: %s, %w", dbName, err)
+	}
+	defer mo.Close()
+
+	MustExec(mo, "DROP DATABASE IF EXISTS "+dbName)
+	MustExec(mo, "CREATE DATABASE "+dbName)
+	MustExec(mo, "USE "+dbName)
+
+	for _, tableInfo := range dbInfo.TableInfos {
+		_, err = mo.Exec(tableInfo.Sql)
+		if err != nil {
+			return fmt.Errorf("failed to create db %s, table %s: %w", dbName, tableInfo.Name, err)
+		}
+	}
+	return nil
+}

t2sql/t2sql/common/spider2_test.go

@@ -69,25 +69,10 @@ func TestCreateMoTables(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	f1db := dbInfos["f1"]
-	if f1db == nil {
-		t.Fatal("f1db not found")
-	}
-
-	mo, err := OpenMoDB()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer mo.Close()
-
-	MustExec(mo, "DROP DATABASE IF EXISTS f1")
-	MustExec(mo, "CREATE DATABASE f1")
-	MustExec(mo, "USE f1")
-
-	for _, tableInfo := range f1db.TableInfos {
-		_, err = mo.Exec(tableInfo.Sql)
+	for _, dbInfo := range dbInfos {
+		t.Logf("Creating tables for %s\n", dbInfo.Name)
+		err = Spider2CreateMoTables(dbInfo)
 		if err != nil {
-			t.Logf("Error creating table %s:\n%s\n", tableInfo.Name, tableInfo.Sql)
 			t.Fatal(err)
 		}
 	}