Should I invoke a separate goroutine for any async operation? - mysql

I have just started with Go with a background in NodeJS. I wrote the following program that opens the connection with the database, queries the database table and prints the record.
package main
import (
"fmt"
"github.com/gin-gonic/gin"
_ "github.com/go-sql-driver/mysql"
"github.com/jinzhu/gorm"
"io"
"net/http"
)
type User struct {
gorm.Model
Username string
Password string
}
func main() {
db, err := gorm.Open("mysql","root:1234#(localhost)/database_test")
db.LogMode(true)
defer db.Close()
if err != nil {
panic("failed to connect to the database")
}
var users []User
db.Table("user").Find(&users)
fmt.Println("after finding the user")
fmt.Println("user ", users)
}
The code works fine. But I want to understand this, doesn't this program block the main thread? Should I be doing any async operation like the database call made inside main in a separate goroutine?
I have been using Promises with NodeJS so it feels a little different with Go.
EDIT: After adding goroutine in the main method
The following code invokes a separate goroutine to query the database. But does not it still block the main thread in that it is waiting for the response from the channel? If yes, is there a better to do this?
package main
import (
"fmt"
"github.com/gin-gonic/gin"
_ "github.com/go-sql-driver/mysql"
"github.com/jinzhu/gorm"
"io"
"net/http"
)
type User struct {
gorm.Model
Username string
Password string
}
func dbRead(db *gorm.DB) <-chan []User {
c := make(chan []User)
go func() {
fmt.Println("starting the go routine to find the users")
defer close(c)
var users []User
db.Table("user").Find(&users)
c <- users
}()
return c
}
func main() {
db, err := gorm.Open("mysql","root:1234#(localhost)/database_test")
db.LogMode(true)
defer db.Close()
if err != nil {
panic("failed to connect to the database")
}
userList := <-dbRead(db)
fmt.Println("user ", userList)
}

Related

Golang - How to create helper mysql to generate uuid / uuid_short without create new connection? Here's the code

I'm trying to generate UUID_SHORT() to be an ID. Instead of using trigger, I get the UUID_SHORT() first and then insert as an ID. But, I'm confusing of how to create GetUUID() function as helper.
As my code bellow, it always setup new connection before generate the UUID_SHORT() which means there will be so many connection just to generate UUID_SHORT()
How to create func GetUUID() without creating new connection?
Here's the code:
package database
import (
"database/sql"
"fmt"
"log"
"os"
"time"
_ "github.com/go-sql-driver/mysql"
)
type conncetion struct {
sqlDB *sql.DB
}
func NewMysqlConnection(databaseConnection *sql.DB) *conncetion {
return &conncetion{databaseConnection}
}
func SetupMysqlDatabaseConnection() (db *sql.DB) {
var (
driver = os.Getenv("DB_DRIVERNAME")
username = os.Getenv("DB_USERNAME")
password = os.Getenv("DB_PASSWORD")
host = os.Getenv("DB_HOST")
port = os.Getenv("DB_PORT")
name = os.Getenv("DB_NAME")
)
connection := fmt.Sprintf("%s:%s#tcp(%s:%s)/%s?parseTime=true", username, password, host, port, name)
db, err := sql.Open(driver, connection)
if err != nil {
log.Fatal(err)
}
db.SetMaxOpenConns(100)
db.SetMaxIdleConns(100)
db.SetConnMaxLifetime(100 * time.Millisecond)
return
}
func (c *conncetion) GenerateUUID() (uuid uint64, err error) {
uuid = 0
queryGetUUID := c.sqlDB.QueryRow(`SELECT UUID_SHORT()`)
err = queryGetUUID.Scan(
&uuid,
)
return
}
func GetUUID() (uuid uint64, err error) {
mysql := SetupMysqlDatabaseConnection()
db := NewMysqlConnection(mysql)
uuid, err = db.GenerateUUID()
return
}
How about global variable ?
package database
import (
"database/sql"
"fmt"
"log"
"os"
"time"
_ "github.com/go-sql-driver/mysql"
)
type conncetion struct {
sqlDB *sql.DB
}
var globalConnection *conncetion
func GetDB() *conncetion {
return globalConnection
}
func NewMysqlConnection(databaseConnection *sql.DB) *conncetion {
return &conncetion{databaseConnection}
}
func SetupMysqlDatabaseConnection() (db *sql.DB) {
var (
driver = os.Getenv("DB_DRIVERNAME")
username = os.Getenv("DB_USERNAME")
password = os.Getenv("DB_PASSWORD")
host = os.Getenv("DB_HOST")
port = os.Getenv("DB_PORT")
name = os.Getenv("DB_NAME")
)
connection := fmt.Sprintf("%s:%s#tcp(%s:%s)/%s?parseTime=true", username, password, host, port, name)
db, err := sql.Open(driver, connection)
if err != nil {
log.Fatal(err)
}
db.SetMaxOpenConns(100)
db.SetMaxIdleConns(100)
db.SetConnMaxLifetime(100 * time.Millisecond)
return
}
func (c *conncetion) GenerateUUID() (uuid uint64, err error) {
uuid = 0
queryGetUUID := c.sqlDB.QueryRow(`SELECT UUID_SHORT()`)
err = queryGetUUID.Scan(
&uuid,
)
return
}
func GetUUID() (uuid uint64, err error) {
// mysql := SetupMysqlDatabaseConnection()
// db := NewMysqlConnection(mysql)
db = GetDB()
uuid, err = db.GenerateUUID()
return
}
// can be written on main.go / server.go
func init() {
globalConnection = NewMysqlConnection(database.SetupMysqlDatabaseConnection())
}
But I don't know it is good or not to open an idle connection and spamming request on 1 connection.
You're leveraging the database to generated the UUID for you, as long as you're letting the DB do it you'll need a connection to send the query.
You generally have two options here:
let the DB generate the UUID on the fly when inserting your data
generate the UUID in your own code before sending it
Generate ID on insert
This means you need to change the way you operate, your command for inserting data will need to look something like this:
INSERT INTO your_table(id, value)
VALUES (
UUID_SHORT(),
-- other values
);
This will automatically generate the ID for you during the insert, without needing to generate it before.
If you need to know the ID after the insert was performed, you have a few options, like using LAST_INSERT_ID() or querying the data you just created.
See this other question for more info.
Generate ID in code
You can use a package like github.com/google/uuid.
import "github.com/google/uuid"
func GenerateUUID() (uint32, error) {
id, err := uuid.NewRandom()
if err != nil {
return 0, err
}
return id.ID(), nil
}
Note that you can also get a string representation for the UUID, or you can cast it to uint64 easily from uint32.

Do I need to close connection in Gorm? [duplicate]

So far the hardest part of Go has been understanding how to organize code. It seems incredibly simple on it's face but every time I've tried to do anything I run into circular imports or things like "exported func Start returns unexported type models.dbStore, which can be annoying to use".
Using the following code how do I call db.Close() or am I really not understanding how I'm supposed to provide the database to my models. Here's what I've got:
App.go
package app
import (
"database/sql"
// Comment
_ "github.com/mattn/go-sqlite3"
)
var (
// DB The database connection
db *sql.DB
)
// Setup Sets up the many many app settings
func Setup() {
d, err := sql.Open("sqlite3", "./foo.db")
if err != nil {
panic(err)
}
// TODO: How does the DB get closed?
// defer db.Close()
db = d
}
// GetDB Returns a reference to the database
func GetDB() *sql.DB {
return db
}
Users.go
package models
import (
"github.com/proj/org/app"
)
// User struct
type User struct {
ID int
}
// CreateUser Creates a user
func (u *User) CreateUser() (int64, error) {
// For the sake of brevity just make sure you can
// "connect" to the database
if err := app.GetDB().Ping(); err != nil {
panic(err)
}
return 1234, nil
}
main.go
package main
import (
"fmt"
"net/http"
_ "github.com/mattn/go-sqlite3"
"github.com/proj/org/app"
"github.com/proj/org/models"
)
func homeHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "You are home")
}
func subscribeHandler(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Subscribing...")
u := models.User{}
u.CreateUser()
}
func main() {
fmt.Println("Running")
app.Setup()
http.HandleFunc("/", homeHandler)
http.HandleFunc("/subscribe", subscribeHandler)
err := http.ListenAndServe(":9090", nil)
if err != nil {
panic(err)
}
}
I thought about doing a app.Shutdown() but that wouldn't work for my most normal use case which is CTRL-C. It would seem if I don't close the database the DB connections would just grow... Just trying to understand.
It's not necessary to close the DB.
The returned DB is safe for concurrent use by multiple goroutines and
maintains its own pool of idle connections. Thus, the Open function
should be called just once. It is rarely necessary to close a DB.
From: https://golang.org/pkg/database/sql/#Open
When your program exits then any open connection is closed, it does not stay open somewhere in the ether awaiting your program to start again, so do not worry about the connections "growing" when you CTRL-C your app.
If, however, you still want to close it, then you can just export a CloseDB function the same way you do with GetDB.
App.go
// ...
func CloseDB() error {
return db.Close()
}
main.go
// ...
func main() {
// ...
app.Setup()
defer app.CloseDB()
// ...
}

How to open and close database connection

I'm beginner in go and now want to create a big project
my question is where should create connection and where close connection in http requests
now i declare db in main function and use in all requests and don't close connection:
package main
import (
"fmt"
"github.com/jinzhu/gorm"
"github.com/kataras/iris"
"github.com/kataras/iris/context"
"github.com/kataras/iris/middleware/recover"
_ "github.com/go-sql-driver/mysql"
)
var db *gorm.DB
func main() {
port := "8080"
app := iris.New()
app.Configure()
app.Logger().SetLevel("debug")
app.Use(recover.New())
db1, err := gorm.Open("mysql", "root:#/database?charset=utf8&parseTime=True&loc=Local")
//db1, err := gorm.Open("mysql", "payro:AEkCpNhd#/payro_db?charset=utf8&parseTime=True&loc=Local")
if err != nil {
fmt.Println(err.Error())
panic(err)
}
db=db1
app.Get("/", func(i context.Context) {
db.Exec("update tbl1 set col1=''")
i.Next()
})
app.Get("/test", func(i context.Context) {
db.Exec("update tbl2 set col2=''")
i.Next()
})
_ = app.Run(iris.Addr(":"+port), iris.WithConfiguration(iris.Configuration{
//DisableBodyConsumptionOnUnmarshal:true,
}), iris.WithoutServerError(iris.ErrServerClosed))
}
this code is ok?
Your code is perhaps a test/POC code.
In a production project, you could go with a MVC or any other kind of architecture as per your needs.
It would be hard to pinpoint the exact structure of your project.
But at the very least, you would want to create a db package which declares an interface for all DB related interactions.
e.g
type UserDBRepo interface{
AddUser(context.Context, *User)
GetUser(context.Context, uint64)
}
type userDBRepo struct{ //implements UserDBRepo
*sql.DB // or whatever type gorm.Open returns
}
func NewUserDBRepo(db *sql.DB) DBRepo{
return &dbRepo{DB: db}
}
The above basically represents a single RDBMS table for this example.
There could be n such files for n DB tables.
Now call NewUserDBRepo from the main.go and pass this instance to all services that need this DB.

How do I select the database to query when using Cloud SQL through App Engine?

I get the following error:
Could not query db: Error 1046: No database selected
I understand what the error message means. But I haven't been able to find the documentation where it says how to select a database.
Here is my code:
package main
import (
"database/sql"
"fmt"
"log"
"net/http"
"os"
"time"
"google.golang.org/appengine"
_ "github.com/go-sql-driver/mysql"
)
var db *sql.DB
func main() {
var (
connectionName = mustGetenv("CLOUDSQL_CONNECTION_NAME")
user = mustGetenv("CLOUDSQL_USER")
password = os.Getenv("CLOUDSQL_PASSWORD")
)
var err error
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/", user, password, connectionName))
if err != nil {
log.Fatalf("Could not open db: %v", err)
}
http.HandleFunc("/", handler)
appengine.Main()
}
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "text/plain")
rows, err := db.Query("INSERT INTO ping ( ping ) VALUES ( '" + time.Now().Format("2006-01-02 03:04:05") + "' );")
if err != nil {
http.Error(w, fmt.Sprintf("Could not query db: %v", err), 500)
return
}
defer rows.Close()
w.Write([]byte("OK"))
}
func mustGetenv(k string) string {
v := os.Getenv(k)
if v == "" {
log.Panicf("%s environment variable not set.", k)
}
return v
}
It looks like you specified the CONNECTION_NAME, but not the DB_NAME. According to the documentation (scroll down to the "GO > Companion process" section), you should open the connection as:
import (
"github.com/go-sql-driver/mysql"
)
dsn := fmt.Sprintf("%s:%s#tcp(%s)/%s",
dbUser,
dbPassword,
"127.0.0.1:3306",
dbName)
db, err := sql.Open("mysql", dsn)
This piece of code resembles yours, but you did not specify the dbName parameter. Bear in mind that the rest of the configuration should remain the same as you shared in your code, but you should just append the name of your database to the second parameter of the sql.Open() function.
In the example of connecting from App Engine Flexible to Cloud SQL using GO, the same procedure is identified:
db, err = sql.Open("mysql", dbName)
So I guess you should try with this change:
// Old connection opening
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/", user, password, connectionName))
// New connection opening, including dbName
db, err = sql.Open("mysql ", fmt.Sprintf("%s:%s#cloudsql(%s)/%s", user, password, connectionName, dbName))
I am not really familiar with GoLang myself, but according to the documentation, that should work.

Share database connection with packages

I'm new with golang. I'm trying to share mysql database connection in my package, latter maybe in several packages. To skip defining database connection in every package I've created Database package and now I'm trying to get that package, connect to db and use that object in whole package.
I'm using this mysql plugin: github.com/go-sql-driver/mysql
here is my code:
main.go
package main
import (
"log"
"./packages/db" // this is my custom database package
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
var dbType Database.DatabaseType
var db *sql.DB
func main() {
log.Printf("-- entering main...")
dbType := Database.New()
db = dbType.GetDb()
dbType.DbConnect()
delete_test_data()
dbType.DbClose()
}
func delete_test_data(){
log.Printf("-- entering delete_test_data...")
//db.Exec("DELETE FROM test;")
}
packages/db/db.go
package Database
import (
"log"
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
type DatabaseType struct {
DatabaseObject *sql.DB
}
func New()(d *DatabaseType) {
d = new(DatabaseType)
//db.DatabaseObject = db.DbConnect()
return d
}
func (d *DatabaseType) DbConnect() *DatabaseType{
log.Printf("-- entering DbConnect...")
var err error
if d.DatabaseObject == nil {
log.Printf("--------- > Database IS NIL...")
d.DatabaseObject, err = sql.Open("mysql", "...")
if err != nil {
panic(err.Error())
}
err = d.DatabaseObject.Ping()
if err != nil {
panic(err.Error())
}
}
return d
}
func (d *DatabaseType) DbClose(){
log.Printf("-- entering DbClose...")
defer d.DatabaseObject.Close()
}
func (d *DatabaseType) GetDb() *sql.DB{
return d.DatabaseObject
}
Everything is ok and without error until I uncomment this line:
db.Exec("DELETE FROM test;")
Can someone tell me what is correct way to share db connection?
Your dbType.DbConnect() method returns a DatabaseType with an initialized connection, but you're ignoring the return value entirely.
Further - to simplify your code - look at having New(host string) *DB instead of three different functions (New/DbConnect/GetDb) that do the same thing.
e.g.
package datastore
type DB struct {
// Directly embed this
*sql.DB
}
func NewDB(host string) (*DB, error) {
db, err := sql.Open(...)
if err != nil {
return nil, err
}
return &DB{db}, nil
}
package main
var db *datastore.DB
func main() {
var err error
db, err = datastore.NewDB(host)
if err != nil {
log.Fatal(err)
}
err := someFunc()
}
func someFunc() error {
rows, err := db.Exec("DELETE FROM ...")
// Handle the error, parse the result, etc.
}
This reduces the juggling you have to do, and you can still call close on your DB type because it embeds *sql.DB - there's no need to implement your own Close() method.