I want to connect mssql, mysql, postgres database servers with ODBC in Go. I have used this package: https://github.com/alexbrainman/odbc
I could connect my local mssql but I could not connect mysql. It gives this error: "Query Preparation Error SQLDriverConnect: {IM002} [Microsoft][ODBC Driver Manager] Data source name not found and no default driver specified"
My OS is Windows 10. After that I will also try on Ubuntu.
This is my code:
import (
"database/sql"
"fmt"
"log"
_ "database/sql/driver"
_ "github.com/alexbrainman/odbc"
)
func main() {
// conn, err := sql.Open("odbc",
// "uid=sa;pwd=Str0ngPa$$w0rd;port=1433;driver=sql server;server=host.docker.internal;database=master;TDS_Version=7.2;")
dbConn := fmt.Sprintf("driver=mysql;server=%s;port=3306;=database=%s;user=%s;password=%s;TDS_Version=7.2;",
"host.docker.internal", "sys", "root", "secret")
conn, err := sql.Open("odbc", dbConn)
if err != nil {
fmt.Println("Connecting Error")
return
}
defer conn.Close()
fmt.Println("Sql Server Connected")
stmt, err := conn.Prepare("select thirdtest, onetest from firsttable WHERE secondtest >= 5 limit 5")
if err != nil {
fmt.Println("Query Preparation Error", err)
return
}
defer stmt.Close()
fmt.Println("Query Prepared")
// Use db.Query() to send the query to the database. Check errors as usual.
row, err := stmt.Query()
if err != nil {
fmt.Println("Query Error", err)
return
}
defer row.Close()
fmt.Printf("\nResult set 1:\n")
for row.Next() {
var (
thirdtest float64
onetest string
)
fmt.Println("result 1")
err := row.Scan(&onetest, &thirdtest)
if err == nil {
fmt.Println(onetest, thirdtest)
}
}
err = row.Err()
if err != nil {
fmt.Printf("\nFatal: %s\n", err)
}
stmt, err = conn.Prepare("select top 5 database_id, name from sys.databases WHERE database_id >= ?")
if err != nil {
log.Fatal(err)
}
row, err = stmt.Query(1)
if err != nil {
log.Fatal(err)
}
defer row.Close()
fmt.Printf("\nResult set 2:\n")
for row.Next() {
var (
id int
name string
)
if err := row.Scan(&id, &name); err == nil {
fmt.Println(id, name)
} else {
log.Fatal(err)
}
}
err = row.Err()
if err != nil {
log.Fatal(err)
}
fmt.Printf("\nFinished correctly\n")
return
}
Related
I have been trying to execute queries with http://www.github.com/go-sql-driver/mysql and have been following this tutorial: https://tutorialedge.net/golang/golang-mysql-tutorial/
Now this tutorial does everything in 1 custom function, but I've tried to split up opening the database connection and the queries itself in seperate functions. The way I've done this is like this:
package main
import (
"fmt"
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
type SqlResult struct {
id int
}
var db *sql.DB
func main() {
dbConnection:= openDatabaseConnection();
getID(dbConnection);
}
func openDatabaseConnection() *sql.DB {
db,err:= sql.Open("mysql","username:password#tcp(127.0.0.1:3306)/test");
if err != nil {
panic(err.Error())
}
defer db.Close()
return db
}
func getID(db *sql.Db) {
results,err:= db.Query("SELECT id FROM test")
for results.Next() {
var result SqlResult
err=results.Scan(&result.id)
if err != nil {
panic(err.Error())
}
fmt.Print(result.id)
}
}
This doesn't print out anything but also doesn't give an error.
Now when I put my code in 1 function, like this:
package main
import (
"fmt"
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
type SqlResult struct {
id int
}
var db *sql.DB
func main() {
openDatabaseConnection();
}
func openDatabaseConnection() *sql.DB {
db,err:= sql.Open("mysql","username:password#tcp(127.0.0.1:3306)/test");
if err != nil {
panic(err.Error())
}
results,err:= db.Query("SELECT id FROM test")
for results.Next() {
var result SqlResult
err=results.Scan(&result.id)
if err != nil {
panic(err.Error())
}
fmt.Print(result.id)
}
defer db.Close()
return db
}
This returns me all the id's, but I want to split the functionality up. It doesn't seem like opening the database connection everytime I need a new query is the right solution.
Does anyone know what I'm doing wrong in my first example and if so, what would be the right way of the way I want it to be?
"what would be the right way of the way I want it to be?"
package main
import (
"database/sql"
"fmt"
_ "github.com/go-sql-driver/mysql"
)
var db *sql.DB
func main() {
db = openDatabaseConnection()
defer db.Close()
printIds(db)
}
func openDatabaseConnection() *sql.DB {
db, err := sql.Open("mysql", "username:password#tcp(127.0.0.1:3306)/test")
if err != nil {
panic(err)
} else if err := db.Ping(); err != nil {
panic(err)
}
return db
}
func printIds(db *sql.DB) {
rows, err := db.Query("SELECT id FROM test")
if err != nil {
panic(err)
}
defer rows.Close()
for rows.Next() {
var id int
if err := rows.Scan(&id); err != nil {
panic(err)
}
fmt.Println(id)
}
if err := rows.Err(); err != nil {
panic(err)
}
}
I am trying to build a TCP server that loads dataset from a CSV file and provide an interface to query the dataset. TCP server will expose port 4040. CSV file contains the following columns related to corona virus cases:
Cumulative Test Positive
Cumulative Tests Performed
Date
Discharged
Expired
Admitted
Region
Users should be able to connect to the server using NetCat nc localhost 4040 command on Linux/Unix based systems.
Once connected to TCP, the user should be able to communicate with the application by sending queries in JSON format.
{
"query": {
"region": "Sindh"
}
}
{
"query": {
"date": "2020-03-20"
}
}
My server.go
package main
import (
"fmt"
"net"
"os"
"flag"
"log"
"encoding/csv"
"encoding/json"
"bufio"
"io"
"strings"
)
type CovidPatient struct {
Positive string `json:"Covid_Positive"`
Performed string `json:"Coivd_Performed"`
Date string `json:"Covid_Date"`
Discharged string `json:"Covid_Discharged"`
Expired string `json:"Covid_Expired"`
Region string `json:"Covid_Region"`
Admitted string `json:"Covid_Admitted"`
}
type DataRequest struct {
Get string `json:"get"`
}
type DataError struct {
Error string `json:"Covid_error"`
}
func Load(path string) []CovidPatient {
table := make([]CovidPatient, 0)
var patient CovidPatient
file, err := os.Open(path)
if err != nil {
panic(err.Error())
}
defer file.Close()
reader := csv.NewReader(file)
csvData, err := reader.ReadAll()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for _, row := range csvData{
patient.Positive = row[0]
patient.Performed = row[1]
patient.Date = row[2]
patient.Discharged = row[3]
patient.Expired = row[4]
patient.Region = row[5]
patient.Admitted = row[6]
table = append(table, patient)
}
return table
}
func Find(table []CovidPatient, filter string) []CovidPatient {
if filter == "" || filter == "*" {
return table
}
result := make([]CovidPatient, 0)
filter = strings.ToUpper(filter)
for _, cp := range table {
if cp.Date == filter ||
cp.Region == filter ||
strings.Contains(strings.ToUpper(cp.Positive), filter) ||
strings.Contains(strings.ToUpper(cp.Performed), filter) ||
strings.Contains(strings.ToUpper(cp.Date), filter) ||
strings.Contains(strings.ToUpper(cp.Discharged), filter) ||
strings.Contains(strings.ToUpper(cp.Expired), filter) ||
strings.Contains(strings.ToUpper(cp.Region), filter) ||
strings.Contains(strings.ToUpper(cp.Admitted), filter){
result = append(result, cp)
}
}
return result
}
var (
patientsDetail = Load("./covid_final_data.csv")
)
func main(){
var addr string
var network string
flag.StringVar(&addr, "e", ":4040", "service endpoint [ip addr or socket path]")
flag.StringVar(&network, "n", "tcp", "network protocol [tcp,unix]")
flag.Parse()
switch network {
case "tcp", "tcp4", "tcp6", "unix":
default:
fmt.Println("unsupported network protocol")
os.Exit(1)
}
ln, err := net.Listen(network, addr)
if err != nil {
log.Println(err)
os.Exit(1)
}
defer ln.Close()
log.Println("Covid19 Condition in Pakistan")
log.Printf("Service started: (%s) %s\n", network, addr)
for {
conn, err := ln.Accept()
if err != nil {
log.Println(err)
conn.Close()
continue
}
log.Println("Connected to ", conn.RemoteAddr())
go handleConnection(conn)
}
}
func handleConnection(conn net.Conn) {
defer func() {
if err := conn.Close(); err != nil {
log.Println("error closing connection:", err)
}
}()
reader := bufio.NewReaderSize(conn, 4)
for {
buf, err := reader.ReadSlice('}')
if err != nil {
if err != io.EOF {
log.Println("connection read error:", err)
return
}
}
reader.Reset(conn)
var req DataRequest
if err := json.Unmarshal(buf, &req); err != nil {
log.Println("failed to unmarshal request:", err)
cerr, jerr := json.Marshal(DataError{Error: err.Error()})
if jerr != nil {
log.Println("failed to marshal DataError:", jerr)
continue
}
if _, werr := conn.Write(cerr); werr != nil {
log.Println("failed to write to DataError:", werr)
return
}
continue
}
result := Find(patientsDetail, req.Get)
rsp, err := json.Marshal(&result)
if err != nil {
log.Println("failed to marshal data:", err)
if _, err := fmt.Fprintf(conn, `{"data_error":"internal error"}`); err != nil {
log.Printf("failed to write to client: %v", err)
return
}
continue
}
if _, err := conn.Write(rsp); err != nil {
log.Println("failed to write response:", err)
return
}
}
}
This correctly loads the csv and convert it into JSON. But, when I try to run query using NetCat command it return empty JSON element. Kindly guide me where is error.
Guess you want this:
╭─root#DESKTOP-OCDRD7Q ~
╰─# nc localhost 4040
{"get": "Sindh"}
[{"Covid_Positive":"1","Coivd_Performed":"1","Covid_Date":"1","Covid_Discharged":"1","Covid_Expired":"1","Covid_Region":"Sindh","Covid_Admitted":"1"}]
What you should do is just to modify your json request.
package main
import (
"bufio"
"encoding/csv"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"net"
"os"
)
type CovidPatient struct {
Positive string `json:"Covid_Positive"`
Performed string `json:"Coivd_Performed"`
Date string `json:"Covid_Date"`
Discharged string `json:"Covid_Discharged"`
Expired string `json:"Covid_Expired"`
Region string `json:"Covid_Region"`
Admitted string `json:"Covid_Admitted"`
}
type DataRequest struct {
Get CovidPatient `json:"get"`
}
type DataError struct {
Error string `json:"Covid_error"`
}
func Load(path string) []CovidPatient {
table := make([]CovidPatient, 0)
var patient CovidPatient
file, err := os.Open(path)
if err != nil {
panic(err.Error())
}
defer file.Close()
reader := csv.NewReader(file)
csvData, err := reader.ReadAll()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
for _, row := range csvData {
patient.Positive = row[0]
patient.Performed = row[1]
patient.Date = row[2]
patient.Discharged = row[3]
patient.Expired = row[4]
patient.Region = row[5]
patient.Admitted = row[6]
table = append(table, patient)
}
return table
}
func Find(table []CovidPatient, filter CovidPatient) []CovidPatient {
result := make([]CovidPatient, 0)
log.Println(filter, table)
for _, cp := range table {
if filter.Positive == "" {
} else if filter.Positive != cp.Positive {
continue
}
if filter.Performed == "" {
} else if filter.Performed != cp.Performed {
continue
}
if filter.Date == "" {
} else if filter.Date != cp.Date {
continue
}
if filter.Discharged == "" {
} else if filter.Discharged != cp.Discharged {
continue
}
if filter.Expired == "" {
} else if filter.Expired != cp.Expired {
continue
}
if filter.Region == "" {
} else if filter.Region != cp.Region {
continue
}
if filter.Admitted == "" {
} else if filter.Admitted != cp.Admitted {
continue
}
result = append(result, cp)
}
return result
}
var (
patientsDetail = Load("./covid_final_data.csv")
)
func main() {
log.SetFlags(log.Lshortfile | log.Ltime)
var addr string
var network string
flag.StringVar(&addr, "e", ":4040", "service endpoint [ip addr or socket path]")
flag.StringVar(&network, "n", "tcp", "network protocol [tcp,unix]")
flag.Parse()
switch network {
case "tcp", "tcp4", "tcp6", "unix":
default:
fmt.Println("unsupported network protocol")
os.Exit(1)
}
ln, err := net.Listen(network, addr)
if err != nil {
log.Println(err)
os.Exit(1)
}
defer ln.Close()
log.Println("Covid19 Condition in Pakistan")
log.Printf("Service started: (%s) %s\n", network, addr)
for {
conn, err := ln.Accept()
if err != nil {
log.Println(err)
conn.Close()
continue
}
log.Println("Connected to ", conn.RemoteAddr())
go handleConnection(conn)
}
}
func handleConnection(conn net.Conn) {
defer func() {
if err := conn.Close(); err != nil {
log.Println("error closing connection:", err)
}
}()
reader := bufio.NewReaderSize(conn, 100)
for {
buf, err := reader.ReadBytes('|')
if err != nil {
if err != io.EOF {
log.Println("connection read error:", err)
return
}
}
reader.Reset(conn)
var req DataRequest
if err := json.Unmarshal(buf[:len(buf)-1], &req); err != nil {
log.Println("failed to unmarshal request:", string(buf), err)
cerr, jerr := json.Marshal(DataError{Error: err.Error()})
if jerr != nil {
log.Println("failed to marshal DataError:", jerr)
continue
}
if _, werr := conn.Write(cerr); werr != nil {
log.Println("failed to write to DataError:", werr)
return
}
continue
}
result := Find(patientsDetail, req.Get)
rsp, err := json.Marshal(&result)
if err != nil {
log.Println("failed to marshal data:", err)
if _, err := fmt.Fprintf(conn, `{"data_error":"internal error"}`); err != nil {
log.Printf("failed to write to client: %v", err)
return
}
continue
}
if _, err := conn.Write(rsp); err != nil {
log.Println("failed to write response:", err)
return
}
}
}
The query is:
╭─root#DESKTOP-OCDRD7Q ~
╰─# nc localhost 4040 127 ↵
{
"get": {
"Covid_Region": "Sindh",
"Covid_Date": "2020-03-20"
}
}|
[{"Covid_Positive":"1","Coivd_Performed":"1","Covid_Date":"2020-03-20","Covid_Discharged":"1","Covid_Expired":"1","Covid_Region":"Sindh","Covid_Admitted":"1"}]
Inside function handleConnection, the first thing is "read until you find the first }", imagine the user is sending the request:
{ "get": { "Covid_Region": "Sindh", "Covid_Date": "2020-03-20" } }
then that step read:
{ "get": { "Covid_Region": "Sindh", "Covid_Date": "2020-03-20" }
Notice the trailing } is missing, then the json.Unmarshal is trying to unmarshal the query without the last } (which is an invalid json).
This problem can take advantage of JSON streaming decoding, in other words, use json.NewDecoder(r io.Reader) instead of json.Unmarshal. Let me copy and modify the first part of that function:
func handleConnection(conn net.Conn) {
defer func() {
if err := conn.Close(); err != nil {
log.Println("error closing connection:", err)
}
}()
jsonDecoder := json.NewDecoder(conn) // A json decoder read a stream to find a
// valid JSON and stop just the byte
// after the JSON ends. Process can be
// repeated.
for {
var req DataRequest
err := jsonDecoder.Decode(&req)
if err == io.EOF {
log.Println("finish")
return
}
if err != nil {
log.Println("unmarshal:", err)
return
}
result := Find(patientsDetail, req.Get) // Here query the system
// ...
Probably now it works, but you can also take advantage of json streaming to send the response back with a jsonEncoder := json.NewEncoder(conn) before de for loop and sending the request like this:
err := jsonEncoder.Encode(&result)
if err != nil {
log.Println("failed to marshal data:", err)
// ...
continue
}
I'm working on a program that makes a query to MySQL, then for each row, changes something with that row and then update the row.
The problem is that sometimes when performing an update I get a deadlock, I'm not sure if it's because the query isn't releasing the lock by the time I update or if it's something else.
Example of what I'm doing:
const (
selectQuery = `select user_id, original_transaction_id, max(payment_id) as max_payment_id from Payment_Receipt
where auto_renew_status = 1 group by user_id, original_transaction_id having count(*) > 1`
updateQuery = `update Payment_Receipt set auto_renew_status = 0, changed_by = "payment_receipt_condenser",
changed_time = ? where user_id = ? and original_transaction_id = ? and payment_id != ? and auto_renew_status = 1`
)
mysql.go:
func New(db *sql.DB, driver string) (database.Database, error) {
sqlDB := sqlx.NewDb(db, driver)
if err := db.Ping(); err != nil {
return nil, errors.Wrap(err, "connecting to database")
}
selectStmt, err := sqlDB.Preparex(selectQuery)
if err != nil {
return nil, errors.Wrap(err, "preparing select query")
}
updateStmt, err := sqlDB.Preparex(updateQuery)
if err != nil {
return nil, errors.Wrap(err, "preparing update query")
}
return &mysql{
db: sqlDB,
selectStmt: selectStmt,
updateStmt: updateStmt,
}, nil
}
func (m *mysql) Query() (<- chan *database.Row, error) {
rowsChan := make(chan *database.Row)
rows, err := m.selectStmt.Queryx()
if err != nil {
return nil, errors.Wrap(err, "making query")
}
go func() {
defer rows.Close()
defer close(rowsChan)
for rows.Next() {
row := &database.Row{}
if err := rows.StructScan(row); err != nil {
log.WithError(err).WithField("user_id", row.UserID.Int32).Error("scanning row")
}
// change some of the data here
// and put into channel for worker to consume
rowsChan <- row
}
}()
return rowsChan, nil
}
func (m *mysql) Update(row *database.Row) error {
tx, err := m.db.Beginx()
if err != nil {
return errors.Wrap(err, "beginning transaction")
}
if _, err := tx.Stmtx(m.updateStmt).Exec(row.ChangedTime); err != nil {
return errors.Wrap(err, "executing update")
}
if err := tx.Commit(); err != nil {
return errors.Wrap(err, "committing transaction")
}
return nil
}
worker.go
func (w *worker) Run(wg *sync.WaitGroup) {
rowsChan, err := w.db.Query()
if err != nil {
log.WithError(err).Fatal("failed making query")
}
for i := 0; i < w.config.Count(); i++ {
wg.Add(1)
go func() {
defer wg.Done()
for row := range rowsChan {
if err := w.db.Update(row); err != nil {
log.WithError(err).WithField("user_id", row.UserID.Int32).Error("updating row")
}
}
}()
}
}
You could make the results (row) channel from a Query() buffered:
func (m *mysql) Query() (<- chan *database.Row, error) {
rowsChan := make(chan *database.Row, 1000) // <- band-aid fix
// ...
}
This will ensure that the row collector function can write multiple results without waiting for your worker go-routine to read the results. The query operation will complete (provided there are 1000 rows or less), and the update go-routine operations can begin their parallel work.
If this fixes things, then consider putting say an SQL limit on your queries (e.g. LIMIT 1000) to ensure you don't hit deadlock again (if 1000+ records is a real possibility).
Crafting "pagination" style queries to grab the next say 1000 rows, using RowID markers etc. to ensure full coverage of results - all while avoiding locking out any of your update operations.
I have a list of locations and I want to know if it's possible to get weather data from the Google Maps API.
When I've had to do this in Go, I've done something like this:
package main
import (
"bufio"
"context"
"database/sql"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strings"
"sync"
"time"
"googlemaps.github.io/maps"
_ "github.com/lib/pq"
)
var db *sql.DB
var (
apiKey string
dsn string
locationsFile string
)
type HourlyForecast struct {
Type string
Properties Properties
}
type Properties struct {
Periods []Period
}
type Period struct {
Temperature int
}
func main() {
flag.StringVar(&apiKey, "api-key", "", "Google Maps API key")
flag.StringVar(&dsn, "dsn", "", "The database connection string")
flag.StringVar(&locationsFile, "locations", "locations.txt", "The locations file")
flag.Parse()
if apiKey == "" {
apiKey = os.Getenv("GOOGLE_MAPS_API_KEY")
}
if apiKey == "" {
log.Fatal("The --api-key flag or GOOGLE_MAPS_API_KEY env var must be set and non-empty")
}
apiKey = strings.TrimSpace(apiKey)
if dsn == "" {
dsn = os.Getenv("DSN")
}
var err error
db, err = sql.Open("postgres", dsn)
if err != nil {
log.Fatal(err)
}
for {
err := db.Ping()
if err != nil {
log.Println(err.Error())
time.Sleep(3 * time.Second)
continue
}
break
}
data, err := os.Open(locationsFile)
if err != nil {
log.Fatal(err)
}
var locations []string
scanner := bufio.NewScanner(data)
for scanner.Scan() {
locations = append(locations, scanner.Text())
}
var wg sync.WaitGroup
for _, location := range locations {
wg.Add(1)
location := location
go func() {
defer wg.Done()
mc, err := maps.NewClient(maps.WithAPIKey(apiKey))
if err != nil {
log.Fatal(err)
}
r := maps.FindPlaceFromTextRequest{
Input: location,
InputType: maps.FindPlaceFromTextInputTypeTextQuery,
}
response, err := mc.FindPlaceFromText(context.Background(), &r)
if err != nil {
log.Fatal(err)
}
pdr := maps.PlaceDetailsRequest{
PlaceID: response.Candidates[0].PlaceID,
}
log.Printf("retrieving geo coordinates for %s", location)
pdResponse, err := mc.PlaceDetails(context.Background(), &pdr)
if err != nil {
log.Fatal(err)
}
lat := pdResponse.Geometry.Location.Lat
lng := pdResponse.Geometry.Location.Lng
u := fmt.Sprintf("https://api.weather.gov/points/%.4f,%.4f/forecast/hourly", lat, lng)
log.Printf("retrieving weather data for %s (%.4f,%.4f)", location, lat, lng)
request, err := http.NewRequest("GET", u, nil)
if err != nil {
log.Fatal(err)
}
request.Header.Add("User-Agent", "Hightower Weather 1.0")
request.Header.Add("Accept", "application/geo+json")
weatherResponse, err := http.DefaultClient.Do(request)
if err != nil {
log.Fatal(err)
}
data, err := ioutil.ReadAll(weatherResponse.Body)
if err != nil {
log.Fatal(err)
}
weatherResponse.Body.Close()
var forecast HourlyForecast
if err := json.Unmarshal(data, &forecast); err != nil {
log.Fatal(err)
}
log.Printf("setting temperature for %s to %d", location, forecast.Properties.Periods[0].Temperature)
_, err = db.Exec(query, location, forecast.Properties.Periods[0].Temperature)
if err != nil {
log.Fatal(err)
}
}()
}
wg.Wait()
}
var query = `INSERT INTO weather (location, temperature)
VALUES ($1, $2)
ON CONFLICT (location)
DO UPDATE SET temperature = EXCLUDED.temperature;`
I am following the tutorials here and here but I am unable to connect to the test database that came with mySQL installation. I can connect to mySql through the command line. What am I missing? When I run the code below I get the error "cannot ping":
package main
import (
"fmt"
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
func main() {
db, err := sql.Open("mysql", "/test")
if err != nil {
fmt.Println(err)
return
}
defer db.Close()
err = db.Ping()
if err != nil {
fmt.Println("cannot ping")
return
}
}
For example, substitute your MySQL user name and password for the words user and password,
package main
import (
"database/sql"
"fmt"
_ "github.com/go-sql-driver/mysql"
)
func main() {
db, err := sql.Open("mysql", "user:password#/test")
if err != nil {
fmt.Println(err)
return
}
defer db.Close()
err = db.Ping()
if err != nil {
fmt.Println(err)
return
}
fmt.Println("Ping")
}
Output:
Ping