plugins/db/MakeTable.go

623 lines
20 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package db
import (
_ "embed"
"fmt"
"github.com/ssgo/db"
"github.com/ssgo/log"
"github.com/ssgo/u"
"os"
"regexp"
"strings"
"text/template"
)
type TableFieldDesc struct {
Field string
Type string
Null string
Key string
Default string
Extra string
After string
}
type TableKeyDesc struct {
Key_name string
Column_name string
}
type TableField struct {
Name string
Type string
Index string
IndexGroup string
Default string
Comment string
IsNull bool
Extra string
desc string
Null string
}
type ERGroup struct {
Group string
Comment string
Tables []TableStruct
}
type TableStruct struct {
Name string
Comment string
Fields []TableField
}
//go:embed er.html
var erTpl string
func (field *TableField) Parse(tableType string) {
//if field.Index == "autoId" {
// field.Type += " unsigned"
//}
if field.IsNull {
field.Null = "NULL"
} else {
field.Null = "NOT NULL"
}
if tableType == "sqlite3" {
// sqlite3 不能修改字段统一使用NULL
field.Null = "NULL"
if field.Extra == "AUTO_INCREMENT" {
field.Extra = "PRIMARY KEY AUTOINCREMENT"
field.Type = "integer"
field.Null = "NOT NULL"
}
}
a := make([]string, 0)
a = append(a, fmt.Sprintf("`%s` %s", field.Name, field.Type))
if tableType == "mysql" {
lowerType := strings.ToLower(field.Type)
if strings.Contains(lowerType, "varchar") || strings.Contains(lowerType, "text") {
a = append(a, " COLLATE utf8mb4_general_ci")
}
}
//if field.Index == "autoId" {
// a = append(a, " AUTO_INCREMENT")
// field.Index = "pk"
// //a = append(a, " NOT NULL")
//}
//if field.Index == "uniqueId" {
// field.Index = "pk"
// //a = append(a, " NOT NULL")
//}
if field.Extra != "" {
a = append(a, " "+field.Extra)
}
a = append(a, " "+field.Null)
if field.Default != "" {
if strings.Contains(field.Default, "CURRENT_TIMESTAMP") {
a = append(a, " DEFAULT "+field.Default)
} else {
a = append(a, " DEFAULT '"+field.Default+"'")
}
}
if tableType == "sqlite3" {
field.Comment = ""
field.Type = "numeric"
} else if tableType == "mysql" {
if field.Comment != "" {
a = append(a, " COMMENT '"+field.Comment+"'")
}
}
field.desc = strings.Join(a, "")
}
var ddlTableMatcher = regexp.MustCompile("(?is)^\\s*CREATE\\s+TABLE\\s+`?([\\w]+)`?\\s*\\(\\s*(.*?)\\s*\\);?\\s*$")
var ddlFieldMatcher = regexp.MustCompile("(?s)\\s*[`\\[]?([\\w]+)[`\\]]?\\s+\\[?([\\w() ]+)\\]?\\s*(.*?)(,|$)")
var ddlKeyMatcher = regexp.MustCompile("[`\\[]?([\\w]+)[`\\]]?\\s*(,|\\))")
var ddlNotNullMatcher = regexp.MustCompile("(?i)\\s+NOT NULL")
var ddlNullMatcher = regexp.MustCompile("(?i)\\s+NULL")
//var ddlDefaultMatcher = regexp.MustCompile("(?i)\\s+DEFAULT\\s+(.*?)$")
var ddlIndexMatcher = regexp.MustCompile("(?is)^\\s*CREATE\\s+([A-Za-z ]+)\\s+`?([\\w]+)`?\\s+ON\\s+`?([\\w]+)`?\\s*\\(\\s*(.*?)\\s*\\);?\\s*$")
var ddlIndexFieldMatcher = regexp.MustCompile("[`\\[]?([\\w]+)[`\\]]?\\s*(,|$)")
// MakeER 创建ER图文件
func MakeER(groups []ERGroup, outputFile *string, tplFile *string, logger *log.Logger) error {
tplStr := ""
if tplFile == nil || *tplFile == "" {
tplStr = erTpl
} else {
tplStr = *tplFile
}
erOutputFile := "er.html"
if outputFile != nil && *outputFile != "" {
erOutputFile = *outputFile
}
tpl := template.New(erOutputFile).Funcs(template.FuncMap{
"short": func(in string) string {
switch in {
case "NULL":
return "n"
case "NOT NULL":
return "nn"
case "AUTO_INCREMENT":
return "ai"
case "CURRENT_TIMESTAMP":
return "ct"
case "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP":
return "ctu"
}
return in
},
})
var err error
tpl, err = tpl.Parse(tplStr)
if err == nil {
var fp *os.File
fp, err = os.OpenFile(erOutputFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err == nil {
err = tpl.Execute(fp, map[string]interface{}{
"title": "ER",
"groups": groups,
})
_ = fp.Close()
}
}
if err != nil {
logger.Error(err.Error())
}
return err
}
func MakeTable(conn *db.DB, table *TableStruct, logger *log.Logger) ([]string, error) {
//fmt.Println(u.JsonP(ddlKeyMatcher.FindAllStringSubmatch("(`key`,id, `name` )", 100)), "====================")
fieldSets := make([]string, 0)
//fieldSetBy := make(map[string]string)
pks := make([]string, 0)
keySets := make([]string, 0)
keySetBy := make(map[string]string)
keySetFields := make(map[string]string)
for i, field := range table.Fields {
field.Parse(conn.Config.Type)
table.Fields[i] = field
if conn.Config.Type == "sqlite3" {
if field.Index == "pk" && field.Extra != "PRIMARY KEY AUTOINCREMENT" {
// sqlite3 用 unique 代替 pk
field.Index = "unique"
field.IndexGroup = "0"
field.Null = "NULL"
}
}
switch field.Index {
case "pk":
if conn.Config.Type == "sqlite3" {
if field.Extra != "PRIMARY KEY AUTOINCREMENT" {
pks = append(pks, field.Name)
}
} else {
pks = append(pks, field.Name)
}
case "unique":
keyName := fmt.Sprint("uk_", table.Name, "_", field.Name)
if field.IndexGroup != "" {
keyName = fmt.Sprint("uk_", table.Name, "_", field.IndexGroup)
}
if keySetBy[keyName] != "" {
keySetFields[keyName] += " " + field.Name
// 复合索引
if conn.Config.Type == "sqlite3" {
keySetBy[keyName] = strings.Replace(keySetBy[keyName], ")", ", `"+field.Name+"`)", 1)
} else if conn.Config.Type == "mysql" {
keySetBy[keyName] = strings.Replace(keySetBy[keyName], ") COMMENT", ", `"+field.Name+"`) COMMENT", 1)
}
} else {
keySetFields[keyName] = field.Name
keySet := ""
if conn.Config.Type == "sqlite3" {
keySet = fmt.Sprintf("CREATE UNIQUE INDEX `%s` ON `%s` (`%s`)", keyName, table.Name, field.Name)
} else if conn.Config.Type == "mysql" {
keySet = fmt.Sprintf("UNIQUE KEY `%s` (`%s`) COMMENT '%s'", keyName, field.Name, field.Comment)
}
keySets = append(keySets, keySet)
keySetBy[keyName] = keySet
}
case "fulltext":
if conn.Config.Type == "mysql" {
keyName := fmt.Sprint("tk_", table.Name, "_", field.Name)
keySet := fmt.Sprintf("FULLTEXT KEY `%s` (`%s`) COMMENT '%s'", keyName, field.Name, field.Comment)
keySets = append(keySets, keySet)
keySetBy[keyName] = keySet
}
case "index":
keyName := fmt.Sprint("ik_", table.Name, "_", field.Name)
if field.IndexGroup != "" {
keyName = fmt.Sprint("ik_", table.Name, "_", field.IndexGroup)
}
if keySetBy[keyName] != "" {
keySetFields[keyName] += " " + field.Name
// 复合索引
if conn.Config.Type == "sqlite3" {
keySetBy[keyName] = strings.Replace(keySetBy[keyName], ")", ", `"+field.Name+"`)", 1)
} else if conn.Config.Type == "mysql" {
keySetBy[keyName] = strings.Replace(keySetBy[keyName], ") COMMENT", ", `"+field.Name+"`) COMMENT", 1)
}
} else {
keySetFields[keyName] = field.Name
keySet := ""
if conn.Config.Type == "sqlite3" {
keySet = fmt.Sprintf("CREATE INDEX `%s` ON `%s` (`%s`)", keyName, table.Name, field.Name)
} else if conn.Config.Type == "mysql" {
keySet = fmt.Sprintf("KEY `%s` (`%s`) COMMENT '%s'", keyName, field.Name, field.Comment)
}
keySets = append(keySets, keySet)
keySetBy[keyName] = keySet
}
}
fieldSets = append(fieldSets, field.desc)
//fieldSetBy[field.Name] = field.desc
}
//fmt.Println(u.JsonP(table.Fields))
//fmt.Println(u.JsonP(keySetBy), 3)
//fmt.Println(u.JsonP(keySets), 4)
outSql := make([]string, 0)
var result *db.ExecResult
var tableInfo map[string]interface{}
if conn.Config.Type == "sqlite3" {
tableInfo = conn.Query("SELECT `name`, `sql` FROM `sqlite_master` WHERE `type`='table' AND `name`='" + table.Name + "'").MapOnR1()
tableInfo["comment"] = ""
} else if conn.Config.Type == "mysql" {
tableInfo = conn.Query("SELECT TABLE_NAME name, TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA='" + conn.Config.DB + "' AND TABLE_NAME='" + table.Name + "'").MapOnR1()
}
oldTableComment := u.String(tableInfo["comment"])
if tableInfo["name"] != nil && tableInfo["name"] != "" {
// 合并字段
oldFieldList := make([]*TableFieldDesc, 0)
oldFields := make(map[string]*TableFieldDesc)
oldIndexes := make(map[string]string)
oldIndexInfos := make([]*TableKeyDesc, 0)
oldComments := map[string]string{}
if conn.Config.Type == "sqlite3" {
tableM := ddlTableMatcher.FindStringSubmatch(u.String(tableInfo["sql"]))
if tableM != nil {
fieldsM := ddlFieldMatcher.FindAllStringSubmatch(tableM[2], 2000)
if fieldsM != nil {
for _, m := range fieldsM {
if m[1] == "PRIMARY" && m[2] == "KEY" {
keysM := ddlKeyMatcher.FindAllStringSubmatch(m[3], 20)
if keysM != nil {
for _, km := range keysM {
oldIndexInfos = append(oldIndexInfos, &TableKeyDesc{
Key_name: "PRIMARY",
Column_name: km[1],
})
}
}
} else {
Null := "NULL"
//fmt.Println(" =====", m[0], m[1], m[2])
if ddlNotNullMatcher.MatchString(m[2]) {
m[2] = ddlNotNullMatcher.ReplaceAllString(m[2], "")
Null = "NOT NULL"
} else if ddlNullMatcher.MatchString(m[2]) {
m[2] = ddlNullMatcher.ReplaceAllString(m[2], "")
Null = "NULL"
}
//fmt.Println(" =====", m[2], "|", Null)
oldFieldList = append(oldFieldList, &TableFieldDesc{
Field: m[1],
Type: m[2],
//Null: u.StringIf(strings.Contains(m[3], "NOT NULL"), "NO", "YES"),
Null: u.StringIf(Null == "NOT NULL", "NO", "YES"),
Key: "",
Default: "",
Extra: "",
After: "",
})
}
}
}
//fmt.Println(u.JsonP(fieldsM), 222)
}
// 读取索引信息
for _, indexInfo := range conn.Query("SELECT `name`, `sql` FROM `sqlite_master` WHERE `type`='index' AND `tbl_name`='" + table.Name + "'").StringMapResults() {
//fmt.Println(u.JsonP(indexInfo), 777)
indexM := ddlIndexMatcher.FindStringSubmatch(indexInfo["sql"])
if indexM != nil {
//fmt.Println(u.JsonP(indexM), 666)
indexFieldM := ddlIndexFieldMatcher.FindAllStringSubmatch(indexM[4], 20)
//fmt.Println(u.JsonP(indexFieldM), 555)
if indexFieldM != nil {
for _, km := range indexFieldM {
oldIndexInfos = append(oldIndexInfos, &TableKeyDesc{
Key_name: indexInfo["name"],
Column_name: km[1],
})
}
}
}
}
//fmt.Println(u.JsonP(oldFieldList), 1)
//fmt.Println(u.JsonP(oldIndexInfos), 2)
} else if conn.Config.Type == "mysql" {
conn.Query("SELECT column_name, column_comment FROM information_schema.columns WHERE TABLE_SCHEMA='" + conn.Config.DB + "' AND TABLE_NAME='" + table.Name + "'").ToKV(&oldComments)
_ = conn.Query("DESC `" + table.Name + "`").To(&oldFieldList)
_ = conn.Query("SHOW INDEX FROM `" + table.Name + "`").To(&oldIndexInfos)
}
//fmt.Println(u.JsonP(oldComments), 111)
for _, indexInfo := range oldIndexInfos {
if oldIndexes[indexInfo.Key_name] == "" {
oldIndexes[indexInfo.Key_name] = indexInfo.Column_name
} else {
oldIndexes[indexInfo.Key_name] += " " + indexInfo.Column_name
}
}
//fmt.Println(u.JsonP(oldIndexes), 111)
//fmt.Println(u.JsonP(keySetFields), 222)
//fmt.Println(u.JsonP(keySetBy), 333)
// 先后顺序
prevFieldId := ""
for _, field := range oldFieldList {
if conn.Config.Type == "sqlite3" {
field.Type = "numeric"
} else if conn.Config.Type == "mysql" {
field.After = prevFieldId
}
prevFieldId = field.Field
oldFields[field.Field] = field
}
//fmt.Println(111, u.JsonP(oldFields), 111)
actions := make([]string, 0)
for keyId := range oldIndexes {
if keyId != "PRIMARY" && strings.ToLower(keySetFields[keyId]) != strings.ToLower(oldIndexes[keyId]) {
if conn.Config.Type == "sqlite3" {
actions = append(actions, "DROP INDEX `"+keyId+"`")
} else if conn.Config.Type == "mysql" {
actions = append(actions, "DROP KEY `"+keyId+"`")
}
}
}
//fmt.Println(" =>>>>>>>>", oldIndexes, pks)
if oldIndexes["PRIMARY"] != "" && strings.ToLower(oldIndexes["PRIMARY"]) != strings.ToLower(strings.Join(pks, " ")) {
if conn.Config.Type == "sqlite3" {
} else if conn.Config.Type == "mysql" {
actions = append(actions, "DROP PRIMARY KEY")
}
}
//for fieldId, fieldSet := range fieldSetBy {
newFieldExists := map[string]bool{}
prevFieldId = ""
for _, field := range table.Fields {
newFieldExists[field.Name] = true
oldField := oldFields[field.Name]
// 修复部分数据库的特殊性
if oldField == nil {
if conn.Config.Type == "sqlite3" {
actions = append(actions, "ALTER TABLE `"+table.Name+"` ADD COLUMN "+field.desc)
} else if conn.Config.Type == "mysql" {
actions = append(actions, "ADD COLUMN "+field.desc)
}
} else {
oldField.Type = strings.TrimSpace(strings.ReplaceAll(oldField.Type, " (", "("))
fixedOldDefault := u.String(oldField.Default)
if fixedOldDefault == "CURRENT_TIMESTAMP" && strings.Contains(oldField.Extra, "on update CURRENT_TIMESTAMP") {
fixedOldDefault = "CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"
}
fixedOldNull := "NOT NULL"
if oldField.Null == "YES" {
fixedOldNull = "NULL"
}
//fmt.Println(" ==", field.Type, "!=", oldField.Type, "||", field.Default, "!=", fixedOldDefault, "||", field.Null, "!=", fixedOldNull, "||", oldField.After, "!=", prevFieldId, "||", oldComments[field.Name], "!=", field.Comment)
//fmt.Println(" ==", strings.ToLower(field.Type) != strings.ToLower(oldField.Type), strings.ToLower(field.Default) != strings.ToLower(fixedOldDefault), strings.ToLower(field.Null) != strings.ToLower(fixedOldNull), strings.ToLower(oldField.After) != strings.ToLower(prevFieldId), strings.ToLower(oldComments[field.Name]) != strings.ToLower(field.Comment))
if strings.ToLower(field.Type) != strings.ToLower(oldField.Type) || strings.ToLower(field.Default) != strings.ToLower(fixedOldDefault) || strings.ToLower(field.Null) != strings.ToLower(fixedOldNull) || strings.ToLower(oldField.After) != strings.ToLower(prevFieldId) || strings.ToLower(oldComments[field.Name]) != strings.ToLower(field.Comment) {
//fmt.Println(" > > > > ", u.JsonP(oldField), 1111)
// `t4f34` varchar(100) COLLATE utf8mb4_general_ci COMMENT ''
// f34, varchar(100), YES, , ,
//fmt.Println(111111, u.JsonP(field), 1111)
// 为什么Desc是空
after := ""
if conn.Config.Type == "mysql" {
if oldField.After != prevFieldId {
if prevFieldId == "" {
after = " FIRST"
} else {
after = " AFTER `" + prevFieldId + "`"
}
}
}
//DROP INDEX `uk_config_key`;
//ALTER TABLE `config` RENAME COLUMN `key` TO `keyOld`;
//ALTER TABLE `config` ADD COLUMN `key` varchar(30) NULL;
//UPDATE `config` SET `key`=`keyOld`;
//ALTER TABLE `config` DROP COLUMN `keyOld`;
//CREATE INDEX `uk_config_key` ON `config` (`key`);
if conn.Config.Type == "sqlite3" {
// 不支持修改字段,所以要先创建然后复制数据再删除
// 方案一(已放弃)重新创建表实现修改
//actions = append(actions, fmt.Sprintf("CREATE TABLE `%s_temp` (\n%s\n)", table.Name, strings.Join(fieldSets, ",\n")))
//actions = append(actions, fmt.Sprintf("INSERT INTO `%s_temp` SELECT * FROM `%s`", table.Name, table.Name))
//actions = append(actions, fmt.Sprintf("DROP TABLE `%s`", table.Name))
//actions = append(actions, fmt.Sprintf("ALTER TABLE `%s_temp` RENAME TO `%s`", table.Name, table.Name))
//INSERT INTO t1_new SELECT foo, bar, baz FROM t1;
//DROP TABLE t1;
//ALTER TABLE t1_new RENAME TO t1;
// 方案二已放弃创建新字段复制数据后删除部分Sqlite不支持DROP COLUMN
//redoIndexes := make([]string, 0)
//for oldIndexName, oldIndex := range oldIndexes {
// if u.StringIn(strings.Split(oldIndex, " "), field.Name) {
// indexSql := conn.Query("SELECT `sql` FROM `sqlite_master` WHERE `type`='index' AND `name`='" + oldIndexName + "'").StringOnR1C1()
// redoIndexes = append(redoIndexes, indexSql)
// actions = append(actions, "DROP INDEX `"+oldIndexName+"`")
// }
//}
//oldPostfix := u.UniqueId()
//actions = append(actions, "ALTER TABLE `"+table.Name+"` RENAME COLUMN `"+field.Name+"` TO `d_"+field.Name+"_"+oldPostfix+"`")
//actions = append(actions, "ALTER TABLE `"+table.Name+"` ADD COLUMN "+field.Desc+after)
//actions = append(actions, "UPDATE `"+table.Name+"` SET `"+field.Name+"`=`d_"+field.Name+"_"+oldPostfix+"`")
////actions = append(actions, "ALTER TABLE `"+table.Name+"` DROP COLUMN `"+field.Name+"Old`")
//for _, redoIndex := range redoIndexes {
// actions = append(actions, redoIndex)
//}
// 方案三 不修改字段类型Sqlite可以兼容
//actions = append(actions, "ALTER TABLE `"+table.Name+"` ADD COLUMN "+field.desc)
} else if conn.Config.Type == "mysql" {
actions = append(actions, "CHANGE `"+field.Name+"` "+field.desc+after)
}
}
}
if conn.Config.Type == "mysql" {
prevFieldId = field.Name
}
}
for oldFieldName := range oldFields {
if newFieldExists[oldFieldName] != true {
if conn.Config.Type == "sqlite3" {
//actions = append(actions, "ALTER TABLE `"+table.Name+"` DROP COLUMN `"+oldFieldName+"`")
} else if conn.Config.Type == "mysql" {
actions = append(actions, "DROP COLUMN `"+oldFieldName+"`")
}
}
}
// sqlite3 不支持添加主键
if conn.Config.Type == "mysql" {
if len(pks) > 0 && strings.ToLower(oldIndexes["PRIMARY"]) != strings.ToLower(strings.Join(pks, " ")) {
actions = append(actions, "ADD PRIMARY KEY(`"+strings.Join(pks, "`,`")+"`)")
}
}
//fmt.Println(111, u.JsonP(oldIndexes), 222 )
//fmt.Println(222, u.JsonP(keySetBy), 222 )
for keyId, keySet := range keySetBy {
if oldIndexes[keyId] == "" || strings.ToLower(oldIndexes[keyId]) != strings.ToLower(keySetFields[keyId]) {
if conn.Config.Type == "sqlite3" {
actions = append(actions, keySet)
} else if conn.Config.Type == "mysql" {
actions = append(actions, "ADD "+keySet)
}
}
}
//fmt.Println(" =>", table.Comment, "|", oldTableComment )
if conn.Config.Type == "mysql" {
if table.Comment != oldTableComment {
actions = append(actions, "COMMENT '"+table.Comment+"'")
}
}
if len(actions) == 0 {
// 不需要更新
return outSql, nil
}
tx := conn.Begin()
defer tx.CheckFinished()
if conn.Config.Type == "sqlite3" {
for _, action := range actions {
//fmt.Println(u.Dim("\t" + strings.ReplaceAll(action, "\n", "\n\t")))
outSql = append(outSql, action)
result = tx.Exec(action)
if result.Error != nil {
break
}
}
} else if conn.Config.Type == "mysql" {
sql := "ALTER TABLE `" + table.Name + "` " + strings.Join(actions, "\n,") + ";"
//fmt.Println(u.Dim("\t" + strings.ReplaceAll(sql, "\n", "\n\t")))
outSql = append(outSql, sql)
result = tx.Exec(sql)
}
if result.Error != nil {
_ = tx.Rollback()
} else {
_ = tx.Commit()
}
} else {
// 创建新表
if len(pks) > 0 {
fieldSets = append(fieldSets, "PRIMARY KEY (`"+strings.Join(pks, "`,`")+"`)")
}
indexSets := make([]string, 0) // sqlite3 额外创建索引的sql
if conn.Config.Type == "sqlite3" {
for _, indexSql := range keySetBy {
indexSets = append(indexSets, indexSql)
}
} else if conn.Config.Type == "mysql" {
for _, key := range keySets {
fieldSets = append(fieldSets, key)
}
}
sql := ""
if conn.Config.Type == "sqlite3" {
sql = fmt.Sprintf("CREATE TABLE `%s` (\n %s\n);", table.Name, strings.Join(fieldSets, ",\n "))
} else if conn.Config.Type == "mysql" {
sql = fmt.Sprintf("CREATE TABLE `%s` (\n %s\n) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='%s';", table.Name, strings.Join(fieldSets, ",\n "), table.Comment)
}
tx := conn.Begin()
defer tx.CheckFinished()
//fmt.Println(u.Dim("\t" + strings.ReplaceAll(sql, "\n", "\n\t")))
outSql = append(outSql, sql)
result = tx.Exec(sql)
if result.Error == nil {
if conn.Config.Type == "sqlite3" {
for _, indexSet := range indexSets {
//fmt.Println(indexSet)
//fmt.Println(u.Dim("\t" + strings.ReplaceAll(indexSet, "\n", "\n\t")))
outSql = append(outSql, indexSet)
r := tx.Exec(indexSet)
if r.Error != nil {
result = r
}
}
}
}
if result.Error != nil {
_ = tx.Rollback()
} else {
_ = tx.Commit()
}
}
if result == nil {
return outSql, nil
}
if result.Error != nil {
logger.Error(result.Error.Error())
}
return outSql, result.Error
}