golang mysql bad connection - mysql

$ wrk -c300 -d10m -t8 url
SetMaxOpenConns(100)
When wrk just started, Occasional bad connection error, And the number of mysql PROCESSLIST is less than 100, around 70
I modified the file
database/sql/sql.go
func OpenDB(c driver.Connector) *DB {
ctx, cancel := context.WithCancel(context.Background())
db := &DB{
connector: c,
openerCh: make(chan struct{}, connectionRequestQueueSize),
resetterCh: make(chan *driverConn, 50),
lastPut: make(map[*driverConn]string),
connRequests: make(map[uint64]chan connRequest),
stop: cancel,
}
go db.connectionOpener(ctx)
go db.connectionResetter(ctx)
return db
}
Changed the buffer length of resetterCh
resetterCh: make(chan *driverConn, 100),
Solved the bad connection problem, and the number of mysql PROCESSLIST instantly reached 100
Is there a problem with my solution?
minimal reproducible example
package main
import (
"fmt"
"github.com/jinzhu/gorm"
"net/http"
_ "github.com/go-sql-driver/mysql"
//_ "github.com/jinzhu/gorm/dialects/mysql"
"time"
)
var (
MysqlClient *gorm.DB
)
const (
user = ""
pwd = ""
host = ""
port = ""
dbname = ""
)
func main() {
initMysqlDb()
http.HandleFunc("/", ActionHandler)
http.ListenAndServe(":18001", nil)
}
type Reguserinfo struct {
Rid int `gorm:"primary_key;column:rid;type:int(10) unsigned;not null" json:"rid"`
Username string `gorm:"unique;column:username;type:varchar(64);not null" json:"username"`
}
func initMysqlDb() {
var err error
uri := fmt.Sprintf("%s:%s#tcp(%s:%s)/%s?charset=utf8mb4&parseTime=True&loc=Local&interpolateParams=true",
user, pwd, host, port, dbname)
MysqlClient, err = gorm.Open("mysql", uri)
if err != nil {
fmt.Println("mysql open err", err)
return
}
// SetMaxIdleConns sets the maximum number of connections in the idle connection pool.
MysqlClient.DB().SetMaxIdleConns(10)
// SetMaxOpenConns sets the maximum number of open connections to the database.
MysqlClient.DB().SetMaxOpenConns(100)
// SetConnMaxLifetime sets the maximum amount of time a connection may be reused.
MysqlClient.DB().SetConnMaxLifetime(100 * time.Second)
MysqlClient.LogMode(false)
MysqlClient.SingularTable(true) //表名后默认+s,关闭之
}
func ActionHandler(w http.ResponseWriter, r *http.Request) {
var result *gorm.DB
var registerInfo Reguserinfo
result = MysqlClient.Where("rid = ?", 2).First(&registerInfo)
if result.Error != nil {
fmt.Println("err", result.Error)
}
w.Write([]byte("hello"))
}
shell run wrk -c300 -d10m -t8 http://127.0.0.1:18001/
This example can be reproduced

Related

Golang - How to create helper mysql to generate uuid / uuid_short without create new connection? Here's the code

I'm trying to generate UUID_SHORT() to be an ID. Instead of using trigger, I get the UUID_SHORT() first and then insert as an ID. But, I'm confusing of how to create GetUUID() function as helper.
As my code bellow, it always setup new connection before generate the UUID_SHORT() which means there will be so many connection just to generate UUID_SHORT()
How to create func GetUUID() without creating new connection?
Here's the code:
package database
import (
"database/sql"
"fmt"
"log"
"os"
"time"
_ "github.com/go-sql-driver/mysql"
)
type conncetion struct {
sqlDB *sql.DB
}
func NewMysqlConnection(databaseConnection *sql.DB) *conncetion {
return &conncetion{databaseConnection}
}
func SetupMysqlDatabaseConnection() (db *sql.DB) {
var (
driver = os.Getenv("DB_DRIVERNAME")
username = os.Getenv("DB_USERNAME")
password = os.Getenv("DB_PASSWORD")
host = os.Getenv("DB_HOST")
port = os.Getenv("DB_PORT")
name = os.Getenv("DB_NAME")
)
connection := fmt.Sprintf("%s:%s#tcp(%s:%s)/%s?parseTime=true", username, password, host, port, name)
db, err := sql.Open(driver, connection)
if err != nil {
log.Fatal(err)
}
db.SetMaxOpenConns(100)
db.SetMaxIdleConns(100)
db.SetConnMaxLifetime(100 * time.Millisecond)
return
}
func (c *conncetion) GenerateUUID() (uuid uint64, err error) {
uuid = 0
queryGetUUID := c.sqlDB.QueryRow(`SELECT UUID_SHORT()`)
err = queryGetUUID.Scan(
&uuid,
)
return
}
func GetUUID() (uuid uint64, err error) {
mysql := SetupMysqlDatabaseConnection()
db := NewMysqlConnection(mysql)
uuid, err = db.GenerateUUID()
return
}
How about global variable ?
package database
import (
"database/sql"
"fmt"
"log"
"os"
"time"
_ "github.com/go-sql-driver/mysql"
)
type conncetion struct {
sqlDB *sql.DB
}
var globalConnection *conncetion
func GetDB() *conncetion {
return globalConnection
}
func NewMysqlConnection(databaseConnection *sql.DB) *conncetion {
return &conncetion{databaseConnection}
}
func SetupMysqlDatabaseConnection() (db *sql.DB) {
var (
driver = os.Getenv("DB_DRIVERNAME")
username = os.Getenv("DB_USERNAME")
password = os.Getenv("DB_PASSWORD")
host = os.Getenv("DB_HOST")
port = os.Getenv("DB_PORT")
name = os.Getenv("DB_NAME")
)
connection := fmt.Sprintf("%s:%s#tcp(%s:%s)/%s?parseTime=true", username, password, host, port, name)
db, err := sql.Open(driver, connection)
if err != nil {
log.Fatal(err)
}
db.SetMaxOpenConns(100)
db.SetMaxIdleConns(100)
db.SetConnMaxLifetime(100 * time.Millisecond)
return
}
func (c *conncetion) GenerateUUID() (uuid uint64, err error) {
uuid = 0
queryGetUUID := c.sqlDB.QueryRow(`SELECT UUID_SHORT()`)
err = queryGetUUID.Scan(
&uuid,
)
return
}
func GetUUID() (uuid uint64, err error) {
// mysql := SetupMysqlDatabaseConnection()
// db := NewMysqlConnection(mysql)
db = GetDB()
uuid, err = db.GenerateUUID()
return
}
// can be written on main.go / server.go
func init() {
globalConnection = NewMysqlConnection(database.SetupMysqlDatabaseConnection())
}
But I don't know it is good or not to open an idle connection and spamming request on 1 connection.
You're leveraging the database to generated the UUID for you, as long as you're letting the DB do it you'll need a connection to send the query.
You generally have two options here:
let the DB generate the UUID on the fly when inserting your data
generate the UUID in your own code before sending it
Generate ID on insert
This means you need to change the way you operate, your command for inserting data will need to look something like this:
INSERT INTO your_table(id, value)
VALUES (
UUID_SHORT(),
-- other values
);
This will automatically generate the ID for you during the insert, without needing to generate it before.
If you need to know the ID after the insert was performed, you have a few options, like using LAST_INSERT_ID() or querying the data you just created.
See this other question for more info.
Generate ID in code
You can use a package like github.com/google/uuid.
import "github.com/google/uuid"
func GenerateUUID() (uint32, error) {
id, err := uuid.NewRandom()
if err != nil {
return 0, err
}
return id.ID(), nil
}
Note that you can also get a string representation for the UUID, or you can cast it to uint64 easily from uint32.

insert query not working in golang

In my golang program, the insert MySQL query was not inserting the values. Even the syntax of the query was correct.It returns my error statement ie internal server error
package main
import (
"database/sql"
"net/http"
"log"
"golang.org/x/crypto/bcrypt"
"encoding/json"
_ "github.com/go-sql-driver/mysql"
"fmt"
)
const hashCost = 8
var db *sql.DB
func initDB(){
var err error
// Connect to the db
//you might have to change the connection string to add your database credentials
db, err = sql.Open("mysql",
"root:nfn#tcp(127.0.0.1:3306)/mydb")
if err != nil {
panic(err)
}
}
// Create a struct that models the structure of a user, both in the request body, and in the DB
type Credentials struct {
Password string `json:"password", db:"password"`
Username string `json:"username", db:"username"`
}
func Signup(w http.ResponseWriter, r *http.Request){
// Parse and decode the request body into a new `Credentials` instance
creds := &Credentials{}
err := json.NewDecoder(r.Body).Decode(creds)
if err != nil {
// If there is something wrong with the request body, return a 400 status
w.WriteHeader(http.StatusBadRequest)
return
}
// Salt and hash the password using the bcrypt algorithm
// The second argument is the cost of hashing, which we arbitrarily set as 8 (this value can be more or less, depending on the computing power you wish to utilize)
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(creds.Password), 8)
// Next, insert the username, along with the hashed password into the database
// Sql query not working Here.How to solve this error
if _, err = db.Exec("INSERT INTO users(username,password) VALUES (?,?) ", creds.Username, string(hashedPassword)); err != nil {
// If there is any issue with inserting into the database, return a 500 error
w.WriteHeader(http.StatusInternalServerError)
return
}
// We reach this point if the credentials we correctly stored in the database, and the default status of 200 is sent back
}

How do I select the database to query when using Cloud SQL through App Engine?

I get the following error:
Could not query db: Error 1046: No database selected
I understand what the error message means. But I haven't been able to find the documentation where it says how to select a database.
Here is my code:
package main
import (
"database/sql"
"fmt"
"log"
"net/http"
"os"
"time"
"google.golang.org/appengine"
_ "github.com/go-sql-driver/mysql"
)
var db *sql.DB
func main() {
var (
connectionName = mustGetenv("CLOUDSQL_CONNECTION_NAME")
user = mustGetenv("CLOUDSQL_USER")
password = os.Getenv("CLOUDSQL_PASSWORD")
)
var err error
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/", user, password, connectionName))
if err != nil {
log.Fatalf("Could not open db: %v", err)
}
http.HandleFunc("/", handler)
appengine.Main()
}
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain")
rows, err := db.Query("INSERT INTO ping ( ping ) VALUES ( '" + time.Now().Format("2006-01-02 03:04:05") + "' );")
if err != nil {
http.Error(w, fmt.Sprintf("Could not query db: %v", err), 500)
return
}
defer rows.Close()
w.Write([]byte("OK"))
}
func mustGetenv(k string) string {
v := os.Getenv(k)
if v == "" {
log.Panicf("%s environment variable not set.", k)
}
return v
}
It looks like you specified the CONNECTION_NAME, but not the DB_NAME. According to the documentation (scroll down to the "GO > Companion process" section), you should open the connection as:
import (
"github.com/go-sql-driver/mysql"
)
dsn := fmt.Sprintf("%s:%s#tcp(%s)/%s",
dbUser,
dbPassword,
"127.0.0.1:3306",
dbName)
db, err := sql.Open("mysql", dsn)
This piece of code resembles yours, but you did not specify the dbName parameter. Bear in mind that the rest of the configuration should remain the same as you shared in your code, but you should just append the name of your database to the second parameter of the sql.Open() function.
In the example of connecting from App Engine Flexible to Cloud SQL using GO, the same procedure is identified:
db, err = sql.Open("mysql", dbName)
So I guess you should try with this change:
// Old connection opening
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/", user, password, connectionName))
// New connection opening, including dbName
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/%s", user, password, connectionName, dbName))
I am not really familiar with GoLang myself, but according to the documentation, that should work.

Gin + Golang + DB Connection Pooling

I would like to understand how does GIN ensures that each HTTP request gets a unique DB ( say MySQL ) connection. Here is one example code.
If you see, since 'db' is a global object and therefore, the API router.GET("/person/:age"... gets access to DB.
Now with load, I suppose GIN will have concurrency implemented internally. If yes, then how does it ensures that each request gets a different connection. If no, then it is single threaded imnplementation. Could anyone please correct my understanding.
package main
import (
// "bytes"
"database/sql"
"fmt"
"github.com/gin-gonic/gin"
_ "github.com/go-sql-driver/mysql"
"net/http"
)
func checkErr(err error) {
if err != nil {
panic(err)
} else {
fmt.Println("successful...")
}
}
func main() {
db, err := sql.Open("mysql", "abfl:abfl#tcp(127.0.0.1:3306)/abfl?charset=utf8")
checkErr(err)
defer db.Close()
// make sure connection is available
err = db.Ping()
checkErr(err)
type User struct {
age int
name string
}
router := gin.Default()
// Add API handlers here
// GET a user detail
router.GET("/person/:age", func(c *gin.Context) {
var (
user User
result gin.H
)
age := c.Param("age")
fmt.Println("input age : '%d'", age)
row := db.QueryRow("select age, name from user where age = ?", age)
err = row.Scan(&user.age, &user.name)
fmt.Printf("user : %+v\n", user)
if err != nil {
// If no results send null
result = gin.H{
"user": nil,
"count": 0,
}
} else {
result = gin.H{
"age": user.age,
"name": user.name,
"count": 1,
}
}
c.JSON(http.StatusOK, result)
})
router.Run(":3000")
}
Establishing a new SQL connection for each HTTP request is too heavy and has no sense.
In go there is no user-managable connection pool yet, it is handled internally by go implementation.
sql.DB is ready to be used concurrently, so there is no worry about it.
And GIN has nothing to do with SQL connections at all. It is fully your responsibility to handle queries/transactions properly.

How to ping remote mysql using golang

I am using go to ping some Linux machines.
But it is not work accurate, because it always needs almost 10 seconds to connect the remote mysql.
I use this command:
mysql -u USER -p PASSWORD -h REMOTE_IP
So i want to know how to use linux command to know whether it open or not.
Then I will use golang 's exec.Command to do that .
Here is my ping code:
package main
import (
"errors"
"fmt"
"github.com/tatsushid/go-fastping"
"net"
"net/smtp"
"strings"
"time"
)
func main(){
err := ping("192.168.2.1")
if err != nil{
fmt.Println("WARNING!!!!!!")
}
}
func ping(ip string) error {
p := fastping.NewPinger()
ra, err := net.ResolveIPAddr("ip", ip)
if err != nil {
return err
}
p.AddIPAddr(ra)
found := false
p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
found = true
}
err = p.Run()
if err != nil {
return err
}
if !found {
return errors.New("response error")
}
return nil
}
I assume you want to know if mysql is reachable or not. You can use following example:
package main
import (
"database/sql"
"flag"
"fmt"
"log"
_ "github.com/go-sql-driver/mysql"
)
func main() {
var (
server = flag.String("mysql", "localhost:3306", "mysql server")
user = flag.String("user", "root", "mysql user")
pass = flag.String("password", "", "mysql password")
)
flag.Parse()
db, err := sql.Open("mysql", fmt.Sprintf("%s:%s#tcp(%s)/", *user, *pass, *server))
if err != nil {
log.Fatalln(err)
}
defer db.Close()
err = db.Ping()
if err != nil {
log.Fatalln(err)
} else {
log.Println("mysqld is alive")
}
}
If you just need to check connection to a specific host & port, you can do this as:
package main
import (
"fmt"
"net"
)
const (
host = "192.168.2.1"
port = "3306" // default MySQL port
)
func main() {
conn, err := net.Dial("tcp", host+":"+port)
if err != nil {
panic(err)
}
defer conn.Close()
fmt.Printf("Connected to %s \n", conn.RemoteAddr().String())
}