MySQL experimental parser (#525)

* update go.mod

* go mod tidy

* Let's crush some bugs

* rm build result

* A bunch of broken tests

* Add comment style to parser

* Add syntax error tests

Author: kyleconroy

go.mod

@@ -7,14 +7,11 @@ require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/go-cmp v0.3.0
 	github.com/jinzhu/inflection v1.0.0
-	github.com/kyleconroy/sqlc-testdata v0.0.0-20200512000015-15313bc43553
 	github.com/lfittl/pg_query_go v1.0.0
 	github.com/lib/pq v1.4.0
-	github.com/pingcap/parser v0.0.0-20200218113622-517beb2e39c2
-	github.com/pingcap/tidb v1.1.0-beta.0.20200219045929-1344d6ddd9e7
+	github.com/pingcap/parser v3.1.1+incompatible
+	github.com/pingcap/tidb v1.1.0-beta.0.20200426094148-df4c9f05c49c
 	github.com/spf13/cobra v0.0.5
-	golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 // indirect
-	golang.org/x/sys v0.0.0-20191220220014-0732a990476f // indirect
 	google.golang.org/genproto v0.0.0-20191223191004-3caeed10a8bf // indirect
 	google.golang.org/grpc v1.26.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71

go.sum

@@ -209,8 +209,7 @@ func (d *dinosqlEngine) Result() golang.Generateable {
 }
 
 func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts dinosql.ParserOpts, stderr io.Writer) (golang.Generateable, bool) {
-	switch sql.Engine {
-	case config.EngineMySQL:
+	if sql.Engine == config.EngineMySQL {
 		// Experimental MySQL support
 		q, err := mysql.GeneratePkg(name, sql.Schema, sql.Queries, combo)
 		if err != nil {
@@ -225,48 +224,36 @@ func parse(e Env, name, dir string, sql config.SQL, combo config.CombinedSetting
 			return nil, true
 		}
 		return q, false
+	}
 
-	case config.EnginePostgreSQL, config.EngineXLemon:
-		var eng postgreEngine
-		if e.ExperimentalParser || sql.Engine == config.EngineXLemon {
-			eng = compiler.NewEngine(sql, combo)
-		} else {
-			eng = &dinosqlEngine{}
-		}
-		if err := eng.ParseCatalog(sql.Schema); err != nil {
-			fmt.Fprintf(stderr, "# package %s\n", name)
-			if parserErr, ok := err.(*multierr.Error); ok {
-				for _, fileErr := range parserErr.Errs() {
-					printFileErr(stderr, dir, fileErr)
-				}
-			} else {
-				fmt.Fprintf(stderr, "error parsing schema: %s\n", err)
+	var eng postgreEngine
+	if sql.Engine == config.EnginePostgreSQL && !e.ExperimentalParser {
+		eng = &dinosqlEngine{}
+	} else {
+		eng = compiler.NewEngine(sql, combo)
+	}
+
+	if err := eng.ParseCatalog(sql.Schema); err != nil {
+		fmt.Fprintf(stderr, "# package %s\n", name)
+		if parserErr, ok := err.(*multierr.Error); ok {
+			for _, fileErr := range parserErr.Errs() {
+				printFileErr(stderr, dir, fileErr)
 			}
-			return nil, true
+		} else {
+			fmt.Fprintf(stderr, "error parsing schema: %s\n", err)
 		}
-		if err := eng.ParseQueries(sql.Queries, parserOpts); err != nil {
-			fmt.Fprintf(stderr, "# package %s\n", name)
-			if parserErr, ok := err.(*multierr.Error); ok {
-				for _, fileErr := range parserErr.Errs() {
-					printFileErr(stderr, dir, fileErr)
-				}
-			} else {
-				fmt.Fprintf(stderr, "error parsing queries: %s\n", err)
+		return nil, true
+	}
+	if err := eng.ParseQueries(sql.Queries, parserOpts); err != nil {
+		fmt.Fprintf(stderr, "# package %s\n", name)
+		if parserErr, ok := err.(*multierr.Error); ok {
+			for _, fileErr := range parserErr.Errs() {
+				printFileErr(stderr, dir, fileErr)
 			}
-			return nil, true
-		}
-		return eng.Result(), false
-
-	case config.EngineXDolphin, config.EngineXElephant:
-		r, err := compiler.Run(sql, combo)
-		if err != nil {
-			fmt.Fprintf(stderr, "# package %s\n", name)
-			fmt.Fprintf(stderr, "error: %s\n", err)
-			return nil, true
+		} else {
+			fmt.Fprintf(stderr, "error parsing queries: %s\n", err)
 		}
-		return r, false
-
-	default:
-		panic("invalid engine")
+		return nil, true
 	}
+	return eng.Result(), false
 }

internal/cmd/generate.go

@@ -7,25 +7,20 @@ import (
 	"io/ioutil"
 	"path/filepath"
 	"regexp"
-	"sort"
 	"strings"
 
-	"github.com/kyleconroy/sqlc/internal/codegen/golang"
-	"github.com/kyleconroy/sqlc/internal/config"
-	"github.com/kyleconroy/sqlc/internal/dolphin"
+	"github.com/kyleconroy/sqlc/internal/metadata"
 	"github.com/kyleconroy/sqlc/internal/migrations"
 	"github.com/kyleconroy/sqlc/internal/multierr"
-	"github.com/kyleconroy/sqlc/internal/pg"
-	"github.com/kyleconroy/sqlc/internal/postgresql"
 	"github.com/kyleconroy/sqlc/internal/sql/ast"
 	"github.com/kyleconroy/sqlc/internal/sql/catalog"
 	"github.com/kyleconroy/sqlc/internal/sql/sqlerr"
 	"github.com/kyleconroy/sqlc/internal/sql/sqlpath"
-	"github.com/kyleconroy/sqlc/internal/sqlite"
 )
 
 type Parser interface {
 	Parse(io.Reader) ([]ast.Statement, error)
+	CommentSyntax() metadata.CommentSyntax
 }
 
 // copied over from gen.go
@@ -145,82 +140,3 @@ func parseQueries(p Parser, c *catalog.Catalog, queries []string) (*Result, erro
 		Queries: q,
 	}, nil
 }
-
-// Deprecated.
-func buildResult(c *catalog.Catalog) (*BuildResult, error) {
-	var structs []golang.Struct
-	var enums []golang.Enum
-	for _, schema := range c.Schemas {
-		for _, table := range schema.Tables {
-			s := golang.Struct{
-				Table:   pg.FQN{Schema: schema.Name, Rel: table.Rel.Name},
-				Name:    strings.Title(table.Rel.Name),
-				Comment: table.Comment,
-			}
-			for _, col := range table.Columns {
-				s.Fields = append(s.Fields, golang.Field{
-					Name:    structName(col.Name),
-					Type:    "string",
-					Tags:    map[string]string{"json:": col.Name},
-					Comment: col.Comment,
-				})
-			}
-			structs = append(structs, s)
-		}
-		for _, typ := range schema.Types {
-			switch t := typ.(type) {
-			case *catalog.Enum:
-				var name string
-				if schema.Name == c.DefaultSchema {
-					name = t.Name
-				} else {
-					name = schema.Name + "_" + t.Name
-				}
-				e := golang.Enum{
-					Name:    structName(name),
-					Comment: t.Comment,
-				}
-				for _, v := range t.Vals {
-					e.Constants = append(e.Constants, golang.Constant{
-						Name:  e.Name + enumValueName(v),
-						Value: v,
-						Type:  e.Name,
-					})
-				}
-				enums = append(enums, e)
-			}
-		}
-	}
-	if len(structs) > 0 {
-		sort.Slice(structs, func(i, j int) bool { return structs[i].Name < structs[j].Name })
-	}
-	if len(enums) > 0 {
-		sort.Slice(enums, func(i, j int) bool { return enums[i].Name < enums[j].Name })
-	}
-	return &BuildResult{structs: structs, enums: enums}, nil
-}
-
-func Run(conf config.SQL, combo config.CombinedSettings) (*BuildResult, error) {
-	var c *catalog.Catalog
-	var p Parser
-
-	switch conf.Engine {
-	case config.EngineXLemon:
-		p = sqlite.NewParser()
-		c = catalog.New("main")
-	case config.EngineXDolphin:
-		p = dolphin.NewParser()
-		c = catalog.New("public") // TODO: What is the default database for MySQL?
-	case config.EngineXElephant:
-		p = postgresql.NewParser()
-		c = postgresql.NewCatalog()
-	default:
-		return nil, fmt.Errorf("unknown engine: %s", conf.Engine)
-	}
-
-	if err := parseCatalog(p, c, conf.Schema); err != nil {
-		return nil, err
-	}
-
-	return buildResult(c)
-}

internal/compiler/compile.go

@@ -28,10 +28,10 @@ func NewEngine(conf config.SQL, combo config.CombinedSettings) *Engine {
 	case config.EngineXLemon:
 		e.parser = sqlite.NewParser()
 		e.catalog = catalog.New("main")
-	case config.EngineXDolphin:
+	case config.EngineMySQL, config.EngineXDolphin:
 		e.parser = dolphin.NewParser()
 		e.catalog = catalog.New("public") // TODO: What is the default database for MySQL?
-	case config.EngineXElephant, config.EnginePostgreSQL:
+	case config.EnginePostgreSQL:
 		e.parser = postgresql.NewParser()
 		e.catalog = postgresql.NewCatalog()
 	default:

internal/compiler/engine.go

@@ -52,7 +52,7 @@ func parseQuery(p Parser, c *catalog.Catalog, stmt ast.Node, src string, rewrite
 	if err := validate.FuncCall(c, raw); err != nil {
 		return nil, err
 	}
-	name, cmd, err := metadata.Parse(strings.TrimSpace(rawSQL), metadata.CommentSyntaxDash)
+	name, cmd, err := metadata.Parse(strings.TrimSpace(rawSQL), p.CommentSyntax())
 	if err != nil {
 		return nil, err
 	}

internal/compiler/parse.go

@@ -74,9 +74,8 @@ const (
 	EnginePostgreSQL Engine = "postgresql"
 
 	// Experimental engines
-	EngineXLemon    Engine = "_lemon"
-	EngineXDolphin  Engine = "_dolphin"
-	EngineXElephant Engine = "_elephant"
+	EngineXLemon   Engine = "_lemon"
+	EngineXDolphin Engine = "_dolphin"
 )
 
 type Config struct {

internal/config/config.go

@@ -0,0 +1,127 @@
+package dolphin
+
+import (
+	pcast "github.com/pingcap/parser/ast"
+	"github.com/pingcap/parser/types"
+
+	"github.com/kyleconroy/sqlc/internal/sql/ast"
+	"github.com/kyleconroy/sqlc/internal/sql/ast/pg"
+)
+
+func convertAlterTableStmt(n *pcast.AlterTableStmt) ast.Node {
+	alt := &ast.AlterTableStmt{
+		Table: parseTableName(n.Table),
+		Cmds:  &ast.List{},
+	}
+	for _, spec := range n.Specs {
+		switch spec.Tp {
+		case pcast.AlterTableAddColumns:
+			for _, def := range spec.NewColumns {
+				name := def.Name.String()
+				alt.Cmds.Items = append(alt.Cmds.Items, &ast.AlterTableCmd{
+					Name:    &name,
+					Subtype: ast.AT_AddColumn,
+					Def: &ast.ColumnDef{
+						Colname:   def.Name.String(),
+						TypeName:  &ast.TypeName{Name: types.TypeStr(def.Tp.Tp)},
+						IsNotNull: isNotNull(def),
+					},
+				})
+			}
+
+		case pcast.AlterTableDropColumn:
+			name := spec.OldColumnName.String()
+			alt.Cmds.Items = append(alt.Cmds.Items, &ast.AlterTableCmd{
+				Name:    &name,
+				Subtype: ast.AT_DropColumn,
+				// MissingOk: spec.IfExists,
+			})
+
+		case pcast.AlterTableChangeColumn:
+			// 	spew.Dump("change column", spec)
+
+		case pcast.AlterTableModifyColumn:
+			// 	spew.Dump("modify column", spec)
+
+		case pcast.AlterTableAlterColumn:
+			// 	spew.Dump("alter column", spec)
+
+		case pcast.AlterTableAddConstraint:
+			// 	spew.Dump("add const", spec)
+
+		default:
+			continue
+		}
+	}
+	return alt
+}
+
+func convertCreateTableStmt(n *pcast.CreateTableStmt) ast.Node {
+	create := &ast.CreateTableStmt{
+		Name:        parseTableName(n.Table),
+		IfNotExists: n.IfNotExists,
+	}
+	for _, def := range n.Cols {
+		create.Cols = append(create.Cols, &ast.ColumnDef{
+			Colname:   def.Name.String(),
+			TypeName:  &ast.TypeName{Name: types.TypeStr(def.Tp.Tp)},
+			IsNotNull: isNotNull(def),
+		})
+	}
+	return create
+}
+
+func convertDropTableStmt(n *pcast.DropTableStmt) ast.Node {
+	drop := &ast.DropTableStmt{IfExists: n.IfExists}
+	for _, name := range n.Tables {
+		drop.Tables = append(drop.Tables, parseTableName(name))
+	}
+	return drop
+}
+
+func convertSelectStmt(n *pcast.SelectStmt) ast.Node {
+	var tables []ast.Node
+	visit(n.From, func(n pcast.Node) {
+		name, ok := n.(*pcast.TableName)
+		if !ok {
+			return
+		}
+		tables = append(tables, parseTableName(name))
+	})
+	var cols []ast.Node
+	visit(n.Fields, func(n pcast.Node) {
+		col, ok := n.(*pcast.ColumnName)
+		if !ok {
+			return
+		}
+		cols = append(cols, &ast.ResTarget{
+			Val: &ast.ColumnRef{
+				Name: col.Name.String(),
+			},
+		})
+	})
+	return &pg.SelectStmt{
+		FromClause: &ast.List{Items: tables},
+		TargetList: &ast.List{Items: cols},
+	}
+}
+
+func convert(node pcast.Node) ast.Node {
+	switch n := node.(type) {
+
+	case *pcast.AlterTableStmt:
+		return convertAlterTableStmt(n)
+
+	case *pcast.CreateTableStmt:
+		return convertCreateTableStmt(n)
+
+	case *pcast.DropTableStmt:
+		return convertDropTableStmt(n)
+
+	case *pcast.SelectStmt:
+		return convertSelectStmt(n)
+
+	default:
+		return &ast.TODO{}
+	}
+}