Chisel compiled successfully but can't generate correctly verilog - chisel

I use Chisel write an RISC-V CPU, Chisel code compiled successfully and Firrtl code generate successfully too, but the verilog code just has a module statement.The Verilog files are basically empty.
It generates all module's Firrtl code.When I use Verilator to simulation it, under the test_run_dir fold it is just a 1kb verilog file and an empty VCD file.
Here is the code
package CPUModule
import chisel3._
import chisel3.util._
import chisel3.iotesters.{ChiselFlatSpec, Driver, PeekPokeTester}
import IFUModule._
import IDUModule._
import MemModel._
import EXUModule._
import WBUModule._
class SingleCycleCPU extends Module {
val io = IO(new Bundle {
val in_enable = Input(Bool())
})
val MM = Module(new MemoryModel) // L1D and L1I
val PC = Module(new PromgrameCounter) // pc
val PD = Module(new PlexDecoder) // decoder
val RF = Module(new RegisterFile) // regfile
val ALU = Module(new ALU) // ALU
val AGU = Module(new LSU) // AGU
val WB = Module(new WriteBackUnit) // write back
// L1I
val MM_in_L1I_readen = Wire(Bool())
val MM_in_L1I_readdr = Wire(UInt(32.W))
val MM_out_L1I_readdata = Wire(UInt(32.W))
// L1D
val MM_in_L1D_readen = Wire(Bool())
val MM_in_L1D_readaddr = Wire(UInt(32.W))
val MM_out_L1D_readdata = Wire(UInt(32.W))
val MM_in_L1D_writeen = Wire(Bool())
val MM_in_L1D_writeaddr = Wire(UInt(32.W))
val MM_in_L1D_writedata = Wire(UInt(32.W))
// not use
MM_in_L1I_readen := false.B
MM_in_L1I_readdr := DontCare
MM_in_L1D_readen := false.B
MM_in_L1D_readaddr := DontCare
// MM_out_L1D_readdata := DontCare
MM_in_L1D_writeen := false.B
MM_in_L1D_writeaddr := DontCare
MM_in_L1D_writedata := DontCare
val MM_L1D_FLAG = Wire(Bool()) // L1D enable
MM.io.in_L1I_readenable := MM_in_L1I_readen
MM.io.in_L1I_readaddr := MM_in_L1I_readdr
MM_out_L1I_readdata := MM.io.out_L1I_readdata
MM.io.in_L1D_readenable := MM_in_L1D_readen
MM.io.in_L1D_readaddr := MM_in_L1D_readaddr
MM_out_L1D_readdata := MM.io.out_L1D_readdata
MM.io.in_L1D_writeenable := MM_in_L1D_writeen
MM.io.in_L1D_writeaddr := MM_in_L1D_writeaddr
MM.io.in_L1D_writedata := MM_in_L1D_writedata
// PC
val PC_out_data = Wire(UInt(32.W))
PC.io.in_enable := io.in_enable // 启动PC
PC.io.in_immpcenable := false.B
PC.io.in_immpcnumber := 0.U(32.W)
PC_out_data := PC.io.out_pcnumber
// get inst
when(!MM_in_L1D_readen && !MM_in_L1D_writeen) {
MM_in_L1I_readen := true.B
MM_in_L1I_readdr := PC_out_data
} .otherwise {
MM_in_L1I_readen := false.B
MM_in_L1I_readdr := DontCare
}
// decoder
val PD_out_mircocode = Wire(UInt(32.W))
val PD_out_rs1 = Wire(UInt(5.W))
val PD_out_rs2 = Wire(UInt(5.W))
val PD_out_rd = Wire(UInt(5.W))
val PD_out_immItype = Wire(UInt(12.W))
val PD_out_immStype5 = Wire(UInt(5.W))
val PD_out_immStype7 = Wire(UInt(7.W))
val PD_out_shamt = Wire(UInt(5.W))
val reg_mircocode = RegInit(0.U(32.W))
val reg_rs1 = RegInit(0.U(5.W))
val reg_rs2 = RegInit(0.U(5.W))
val reg_rd = RegInit(0.U(5.W))
val reg_immItype = RegInit(0.U(12.W))
val reg_immStype5 = RegInit(0.U(5.W))
val reg_immStype7 = RegInit(0.U(7.W))
val reg_shamt = RegInit(0.U(5.W))
PD.io.in_instruction := MM_out_L1I_readdata
PD_out_mircocode := PD.io.out_mircocode
PD_out_rs1 := PD.io.out_rs1
PD_out_rs2 := PD.io.out_rs2
PD_out_rd := PD.io.out_rd
PD_out_immItype := PD.io.out_immItype
PD_out_immStype5 := PD.io.out_immStype5
PD_out_immStype7 := PD.io.out_immStype7
PD_out_shamt := PD.io.out_shamt
reg_mircocode := PD_out_mircocode
reg_rs1 := PD_out_rs1
reg_rs2 := PD_out_rs2
reg_rd := PD_out_rd
reg_immItype := PD_out_immItype
reg_immStype5 := PD_out_immStype5
reg_immStype7 := PD_out_immStype7
reg_shamt := PD_out_shamt
// reg file
val RF_in_readen = Wire(Bool())
val RF_out_readdata_1 = Wire(UInt(32.W))
val RF_out_readdata_2 = Wire(UInt(32.W))
val RF_in_writeen = Wire(Bool())
val RF_in_writeaddr = Wire(UInt(32.W))
val RF_in_writedata = Wire(UInt(32.W))
RF_in_readen := true.B
RF_in_writeen := false.B
RF.io.in_read := RF_in_readen
RF.io.in_readaddress_1 := reg_rs1
RF.io.in_readaddress_2 := reg_rs2
RF_out_readdata_1 := RF.io.out_readdata_1
RF_out_readdata_2 := RF.io.out_readdata_2
RF.io.in_write := RF_in_writeen
RF.io.in_writeaddress_1 := RF_in_writeaddr
RF.io.in_writedata_1 := RF_in_writedata
val reg_rf_data_1 = RegInit(0.U(32.W))
val reg_rf_data_2 = RegInit(0.U(32.W))
reg_rf_data_1 := RF_out_readdata_1
reg_rf_data_2 := RF_out_readdata_2
// ALU AGU
// ALU
val ALU_out_rd = Wire(UInt(32.W))
ALU.io.in_mircocode := reg_mircocode
ALU.io.in_rs1data := reg_rf_data_1
ALU.io.in_rs2data := reg_rf_data_2
ALU.io.in_immItype := reg_immItype
ALU.io.in_shamt := reg_shamt
ALU_out_rd := ALU.io.out_rddata
// AGU
val AGU_out_L1D_readen = Wire(Bool())
val AGU_out_L1D_readaddr = Wire(UInt(32.W))
val AGU_out_L1D_writeen = Wire(Bool())
val AGU_out_L1D_writeaddr = Wire(UInt(32.W))
val AGU_out_L1D_writedata = Wire(UInt(32.W))
val AGU_out_rdaddr = Wire(UInt(5.W))
AGU.io.in_mircocode := reg_mircocode
AGU.io.in_rs1data := reg_rf_data_1
AGU.io.in_rs2data := reg_rf_data_2
AGU.io.in_immItype := reg_immItype
AGU.io.in_immStype5 := reg_immStype5
AGU.io.in_immStype7 := reg_immStype7
AGU.io.in_rdaddress := reg_rd
AGU_out_L1D_readen := AGU.io.out_read_enable
AGU_out_L1D_writeen := AGU.io.out_write_enable
AGU_out_L1D_writeaddr := AGU.io.out_writeaddress
AGU_out_L1D_writedata := AGU.io.out_writedata
AGU_out_L1D_readaddr := AGU.io.out_readaddress
AGU_out_rdaddr := AGU.io.out_rdaddress
// connect AGU output to L1D input
when(AGU_out_L1D_readen) {
// connect
MM_in_L1D_readen := AGU_out_L1D_readen
MM_in_L1D_readaddr := AGU_out_L1D_readaddr
// not use
MM_in_L1D_writeen := false.B
MM_in_L1D_writeaddr := DontCare
MM_in_L1D_writedata := DontCare
MM_L1D_FLAG := true.B
} .elsewhen(AGU_out_L1D_writeen) {
// connect
MM_in_L1D_writeen := AGU_out_L1D_writeen
MM_in_L1D_writeaddr := AGU_out_L1D_writeaddr
MM_in_L1D_writedata := AGU_out_L1D_writedata
// not use
MM_in_L1D_readen := false.B
MM_in_L1D_readaddr := DontCare
MM_L1D_FLAG := false.B
} .otherwise {
// not use
MM_in_L1D_readen := false.B
MM_in_L1D_readaddr := DontCare
MM_in_L1D_writeen := false.B
MM_in_L1D_writeaddr := DontCare
MM_in_L1D_writedata := DontCare
MM_L1D_FLAG := true.B
}
val reg_L1D_readdata = RegInit(0.U(32.W))
reg_L1D_readdata := MM_out_L1D_readdata
val reg_ALU_rddata = RegInit(0.U(32.W))
reg_ALU_rddata := ALU_out_rd
val reg_AGU_rdaddr = RegInit(0.U(4.W))
reg_AGU_rdaddr := AGU_out_rdaddr
// write back
val WB_in_enable = Wire(Bool())
val WB_out_rdaddr = Wire(UInt(32.W))
val WB_out_rddata = Wire(UInt(32.W))
WB_in_enable := true.B
WB.io.in_enable := WB_in_enable
WB.io.in_address_1 := reg_AGU_rdaddr
WB.io.in_needwritedata_1 := reg_L1D_readdata
WB.io.in_mircocode := reg_mircocode
WB.io.in_address_2 := reg_rd
WB.io.in_needwritedata_2 := reg_ALU_rddata
WB_out_rdaddr := WB.io.out_address
WB_out_rddata := WB.io.out_data
val reg_wb_addr = RegInit(0.U(5.W))
val reg_wb_data = RegInit(0.U(32.W))
reg_wb_addr := WB_out_rdaddr
reg_wb_data := WB_out_rddata
// close read com
RF_in_readen := false.B
// open write com
RF_in_writeen := true.B
RF_in_writeaddr := reg_wb_addr
RF_in_writedata := reg_wb_data
}
// Tester
class TestSingleCycleCPU(c: SingleCycleCPU) extends PeekPokeTester(c) {
// poke
poke(c.io.in_enable, true.B)
// wait
step(1)
}
object SingleCycleCPU {
def main(args: Array[String]): Unit = {
val args = Array("--backend-name", "verilator")
chisel3.iotesters.Driver.execute(args, () => new SingleCycleCPU) { c => new TestSingleCycleCPU(c) }
// chisel3.Driver.execute(args, () => new SingleCycleCPU)
}
}

The FIRRTL compiler does a fair amount of optimizations between the output of Chisel and emitting Verilog. In this case, your code is getting removed by Dead Code Elimination because there is no effect on the outside world.
I would suggest adding some output to monitor what's going on, perhaps turn PC_out_data into an output:
val io = IO(new Bundle {
val in_enable = Input(Bool())
val PC_out_data = Output(UInt(32.W))
})
You'll have to replace references to PC_out_data with io.PC_out_data, but if you do this, anything that has an effect on the PC will no longer be deleted.
For more information, check out my answer to this question which discusses optimizations and how they remove signals (in addition to how names are propagated from Chisel to Verilog which you also might find interesting): How to keep all variable name In chisel when generate Verilog code

Related

retrieving data from json response from an API

I'm trying to get data from json response by storing it in some structs (Airport + coordinates) but I don't know how to deal with that since I'm not good enough with maps and interfaces. The code is showing no errors but MapofAirports is completely empty here is the code:
package main
import (
//"api/client"
//"api/client/clienterrors"
//"api/client/openstreetmap"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"math"
"net/http"
"os"
"strconv"
"strings"
)
type Coordinates struct {
Longitude string `json:"lon"`
Latitude string `json:"lat"`
}
type Airport struct {
Co Coordinates `json:"location"`
IATACode string `json:"id"`
Distance float64 `json:"distance"` // distance to coordinates in kilometer
}
func GetCoordinatesFromURL(url string) (float64, float64) {
parts := strings.Split(url, "=")
lat0 := strings.Split(parts[2], "&")
lon0 := strings.Split(parts[3], "&")
lat1, _ := strconv.ParseFloat(lat0[0], 64)
lon1, _ := strconv.ParseFloat(lon0[0], 64)
return lat1, lon1
}
func CalcDistance(lat1 float64, long1 float64, lat2 float64, long2 float64) float64 {
var latitude1 = lat1 * math.Pi / 180
var latitude2 = lat2 * math.Pi / 180
var longitude1 = long1 * math.Pi / 180
var longitude2 = long2 * math.Pi / 180
var R = 6371.0
var d = R * math.Acos(math.Cos(latitude1)*math.Cos(latitude2)*math.Cos(longitude2-longitude1)+math.Sin(latitude1)*math.Sin(latitude2))
return d
}
func main() {
var Locations []Airport
Locations = make([]Airport, 0)
var url = fmt.Sprintf("https://api.skypicker.com/locations?type=radius&lat=40.730610&lon=-73.935242&radius=250&location_types=airport&limit=3&sort=id&active_only=true")
UrlLat, UrlLon := GetCoordinatesFromURL(url)
resp, err := http.Get(url)
if err != nil {
panic(err.Error())
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
var airportsJsonResponse interface{}
err = json.Unmarshal(body, &airportsJsonResponse)
MapofAirports, ok := airportsJsonResponse.([]interface{})
if ok {
lenAiroMap := len(MapofAirports)
locationsMaps := make(map[int]map[string]interface{})
for i := 0; i < lenAiroMap; i++ {
locationsMaps[i] = MapofAirports[i].(map[string]interface{})
}
var coords Coordinates
for i := 0; i < lenAiroMap; i++ {
if longitude, ok0 := locationsMaps[i]["lon"].(string); ok0 {
if latitude, ok1 := locationsMaps[i]["lat"].(string); ok1 {
coords = Coordinates{longitude, latitude}
}
}
code := locationsMaps[i]["id"].(string)
latFromCoordinates, _ := strconv.ParseFloat(Locations[i].Co.Latitude, 64)
lonFromCoordinates, _ := strconv.ParseFloat(Locations[i].Co.Longitude, 64)
dist := CalcDistance(latFromCoordinates, lonFromCoordinates, UrlLat, UrlLon)
Locations = append(Locations, Airport{
Co: coords,
IATACode: code,
Distance: dist,
})
}
}
LocationsJson, err := json.Marshal(Locations)
if err != nil {
log.Fatal("Cannot encode to JSON ", err)
}
fmt.Fprintf(os.Stdout, "%s", LocationsJson)
}
screenshot of json response
in the screenshot this is the json response we have, and I'm processing like this:
{ locations[],meta,last_refresh,results_retrieved } ==> location : { id , location + distance(calculated with a function) }
Change this line MapofAirports, ok := airportsJsonResponse.([]interface{}) to this
MapofAirports, ok := airportsJsonResponse.(map[string]interface{})
If you place a break point at this line you will see type of airportsJsonResponse is map[string]interface{}.
And you will have to change this lines into key value iteration
for i := 0; i < lenAiroMap; i++ {
locationsMaps[i] = MapofAirports[i].(map[string]interface{})
}
in to sth like below :
lenAiroMap := len(MapofAirports)
locationsMaps := make([]map[string]interface{},lenAiroMap)
for i, value := range MapofAirports["locations"].([]interface{}) {
converted := value.(map[string]interface{})
locationsMaps[i] = converted
}
this is my last update and while running the program it panics at unmarshaling step
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"math"
"net/http"
"strconv"
"strings"
)
type Coordinates struct {
Longitude string `json:"lon"`
Latitude string `json:"lat"`
}
type Airport struct {
Co Coordinates `json:"location"`
IATACode string `json:"id"`
Distance float64 `json:"distance"` // distance to coordinates in kilometer
}
type Response struct {
Locations []Airport `json:"locations"`
// add all the other fields you care about
}
func GetCoordinatesFromURL(url string) (float64, float64) {
parts := strings.Split(url, "=")
lat0 := strings.Split(parts[2], "&")
lon0 := strings.Split(parts[3], "&")
lat1, _ := strconv.ParseFloat(lat0[0], 64)
lon1, _ := strconv.ParseFloat(lon0[0], 64)
return lat1, lon1
}
func CalcDistance(lat1 float64, long1 float64, lat2 float64, long2 float64) float64 {
var latitude1 = lat1 * math.Pi / 180
var latitude2 = lat2 * math.Pi / 180
var longitude1 = long1 * math.Pi / 180
var longitude2 = long2 * math.Pi / 180
var R = 6371.0
var d = R * math.Acos(math.Cos(latitude1)*math.Cos(latitude2)*math.Cos(longitude2-longitude1)+math.Sin(latitude1)*math.Sin(latitude2))
return d
}
func main() {
var url = fmt.Sprintf("https://api.skypicker.com/locations?type=radius&lat=40.730610&lon=-73.935242&radius=250&location_types=airport&limit=3&sort=id&active_only=true")
UrlLat, UrlLon := GetCoordinatesFromURL(url)
resp, err := http.Get(url)
if err != nil {
panic(err.Error())
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
res := &Response{}
if err := json.Unmarshal(data, res); err != nil {
panic(err)
}
fmt.Println(res.Locations)
for i, item := range res.Locations {
latt,_ := strconv.ParseFloat(item.Co.Latitude, 64)
lonn,_ :=strconv.ParseFloat(item.Co.Longitude, 64)
res.Locations[i].Distance = CalcDistance(latt,lonn , UrlLat, UrlLon)
}
fmt.Println("after calculate distance")
fmt.Println(res.Locations)
}
what's wrong with that ?

How do you instanciate a same module twice?

I am developping a code in chisel, and tried to instanciate a module Encryption twice.
If I just use Enc0 in the code below, it works fine.
But if I use Enc0 and Enc1, then I have the following error appearing for line 40 :
[error] chisel3.internal.ChiselException: Connection between sink (chisel3.core.UInt#1fc1) and source (chisel3.core.UInt#1f8d) failed #: Sink or source unavailable to current module.
30 val Enc0 = Module(new Encryption())
31 Enc0.io.lab1 := a0
32 Enc0.io.lab2 := b0
33 Enc0.io.lab3 := a0 & b0
34 Enc0.io.key := io.secret_key
35 Enc0.io.wire_id := io.wire_index
36 Enc0.io.go := io.go
37 val tab0 = Enc0.io.enc
38 io.garbled_table.out0 := tab0
39
40 val Enc1 = Module(new Encryption())
41 Enc1.io.lab1 := a0
42 Enc1.io.lab2 := b1
43 Enc1.io.lab3 := a0 & b1
44 Enc1.io.key := io.secret_key
45 Enc1.io.wire_id := io.wire_index
46 Enc1.io.go := io.go
47 val tab1 = Enc1.io.enc
48 io.garbled_table.out1 := tab1
All the inputs and outputs of Enc0 and Enc1 are correctly connected, since Enc0 can work when I comment all lines 40-48.
So I don't know why it is not working
I have expanded this out to a module that does compile. Perhaps you can compare this to your example above. It is probably an error in the IO direction of one of your intermediate wires. It's less than ideal (there is working being done to improve error messages like this) but can you figure out which line is the problem by uncommenting replacing the right hand sides of 40-48 with DontCare, and replacing those one by one until you narrow down the offending line.
My example that seems to build.
import chisel3._
import chisel3.experimental.MultiIOModule
class Encryption extends Module {
val io = IO(new Bundle {
val lab1 = Input(Bool())
val lab2 = Input(Bool())
val lab3 = Input(Bool())
val key = Input(UInt(8.W))
val wire_id = Input(UInt(8.W))
val go = Input(UInt(8.W))
val enc = Output(UInt(8.W))
})
}
class Parent extends MultiIOModule {
val a0 = IO(Input(Bool()))
val b0 = IO(Input(Bool()))
val a1 = IO(Input(Bool()))
val b1 = IO(Input(Bool()))
val secret_key = IO(Input(UInt(8.W)))
val io = IO(new Bundle {
val secret_key = Input(UInt(8.W))
val wire_index = Input(UInt(8.W))
val garbled_table = new Bundle {
val out0 = Output(UInt(8.W))
val out1 = Output(UInt(8.W))
}
val go = Input(UInt(8.W))
})
val Enc0 = Module(new Encryption())
Enc0.io.lab1 := a0
Enc0.io.lab2 := b0
Enc0.io.lab3 := a0 & b0
Enc0.io.key := io.secret_key
Enc0.io.wire_id := io.wire_index
Enc0.io.go := io.go
val tab0 = Enc0.io.enc
io.garbled_table.out0 := tab0
val Enc1 = Module(new Encryption())
Enc1.io.lab1 := a0
Enc1.io.lab2 := b1
Enc1.io.lab3 := a0 & b1
Enc1.io.key := io.secret_key
Enc1.io.wire_id := io.wire_index
Enc1.io.go := io.go
val tab1 = Enc1.io.enc
io.garbled_table.out1 := tab1
}
object Encryption {
def main(args: Array[String]): Unit = {
println(Driver.emit(() => new Parent))
}
}

Is there a better way to marshal sql rows?

I have the following struct:
type MyTable struct{
DBColA []byte `db:"cola" json:"-"`
ColA string `json:"cola"`
DBColB []byte `db:"colb" json:"-"`
ColB string `json:"colb"`
}
I map to []byte [to better handle null values in my sql][1]
When I grab the rows I need to output it as json. In order to do that I convert []byte to string:
var rows []*MyTable
if _, err := Session.Select(&rows, sql, args...); err != nil {
log.Println(err)
}
for _, row := range rows{
row.ColA = string(row.DBColA)
row.ColB = string(row.DBColB)
}
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(rows); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
It seems very inefficient to have DBColA and ColA in my struct and then converting DBColA to a string....I have a lot of columns. Is there a better way?
[1]: https://github.com/go-sql-driver/mysql/wiki/Examples
Have you tried gosqljson in https://github.com/elgs/gosqljson ?
See example:
```golang
package main
import (
"database/sql"
"fmt"
"github.com/elgs/gosqljson"
_ "github.com/go-sql-driver/mysql"
)
func main() {
ds := "username:password#tcp(host:3306)/db"
db, err := sql.Open("mysql", ds)
defer db.Close()
if err != nil {
fmt.Println("sql.Open:", err)
}
theCase := "lower" // "lower" default, "upper", camel
a, _ := gosqljson.QueryDbToArrayJson(db, theCase, "SELECT ID,NAME FROM t LIMIT ?,?", 0, 3)
fmt.Println(a)
// [["id","name"],["0","Alicia"],["1","Brian"],["2","Chloe"]]
m, _ := gosqljson.QueryDbToMapJson(db, theCase, "SELECT ID,NAME FROM t LIMIT ?,?", 0, 3)
fmt.Println(m)
// [{"id":"0","name":"Alicia"},{"id":"1","name":"Brian"},{"id":"2","name":"Chloe"}]
}
````

Dumping MySQL tables to JSON with Golang

Was putting together a quick dumper for MySQL to JSON in Go. However I find that everything that I retrieve from the database is a []byte array. Thus instead of native JSON integers or booleans, I'm getting everything encoded as strings.
Subset of the code:
import (
"encoding/json"
"database/sql"
_ "github.com/go-sql-driver/mysql"
)
func dumpTable(w io.Writer, table) {
// ...
rows, err := Query(db, fmt.Sprintf("SELECT * FROM %s", table))
checkError(err)
columns, err := rows.Columns()
checkError(err)
scanArgs := make([]interface{}, len(columns))
values := make([]interface{}, len(columns))
for i := range values {
scanArgs[i] = &values[i]
}
for rows.Next() {
err = rows.Scan(scanArgs...)
checkError(err)
record := make(map[string]interface{})
for i, col := range values {
if col != nil {
fmt.Printf("\n%s: type= %s\n", columns[i], reflect.TypeOf(col))
switch t := col.(type) {
default:
fmt.Printf("Unexpected type %T\n", t)
case bool:
fmt.Printf("bool\n")
record[columns[i]] = col.(bool)
case int:
fmt.Printf("int\n")
record[columns[i]] = col.(int)
case int64:
fmt.Printf("int64\n")
record[columns[i]] = col.(int64)
case float64:
fmt.Printf("float64\n")
record[columns[i]] = col.(float64)
case string:
fmt.Printf("string\n")
record[columns[i]] = col.(string)
case []byte: // -- all cases go HERE!
fmt.Printf("[]byte\n")
record[columns[i]] = string(col.([]byte))
case time.Time:
// record[columns[i]] = col.(string)
}
}
}
s, _ := json.Marshal(record)
w.Write(s)
io.WriteString(w, "\n")
}
}
I also needed to dump database tables to json and here is how I achieved:
(different than another answer in this topic, everything is not string, thanks to this answer: https://stackoverflow.com/a/17885636/4124416, I could get integer fields correctly)
func getJSON(sqlString string) (string, error) {
rows, err := db.Query(sqlString)
if err != nil {
return "", err
}
defer rows.Close()
columns, err := rows.Columns()
if err != nil {
return "", err
}
count := len(columns)
tableData := make([]map[string]interface{}, 0)
values := make([]interface{}, count)
valuePtrs := make([]interface{}, count)
for rows.Next() {
for i := 0; i < count; i++ {
valuePtrs[i] = &values[i]
}
rows.Scan(valuePtrs...)
entry := make(map[string]interface{})
for i, col := range columns {
var v interface{}
val := values[i]
b, ok := val.([]byte)
if ok {
v = string(b)
} else {
v = val
}
entry[col] = v
}
tableData = append(tableData, entry)
}
jsonData, err := json.Marshal(tableData)
if err != nil {
return "", err
}
fmt.Println(string(jsonData))
return string(jsonData), nil
}
Here is a sample output:
[{"ID":0,"Text":"Zero"},{"ID":1,"Text":"One"},{"ID":2,"Text":"Two"}]
It is needed to use prepared statements to get the native types. MySQL has two protocols, one transmits everything as text, the other as the "real" type. And that binary protocol is only used when you use prepared statements. See https://github.com/go-sql-driver/mysql/issues/407
The function getJSON below is correct:
func getJSON(sqlString string) (string, error) {
stmt, err := db.Prepare(sqlString)
if err != nil {
return "", err
}
defer stmt.Close()
rows, err := stmt.Query()
if err != nil {
return "", err
}
defer rows.Close()
columns, err := rows.Columns()
if err != nil {
return "", err
}
tableData := make([]map[string]interface{}, 0)
count := len(columns)
values := make([]interface{}, count)
scanArgs := make([]interface{}, count)
for i := range values {
scanArgs[i] = &values[i]
}
for rows.Next() {
err := rows.Scan(scanArgs...)
if err != nil {
return "", err
}
entry := make(map[string]interface{})
for i, col := range columns {
v := values[i]
b, ok := v.([]byte)
if (ok) {
entry[col] = string(b)
} else {
entry[col] = v
}
}
tableData = append(tableData, entry)
}
jsonData, err := json.Marshal(tableData)
if err != nil {
return "", err
}
return string(jsonData), nil
}
There's not much you can do because the driver - database/sql interaction is pretty much a one way street and the driver can't help you with anything when the data is handed over to database/sql.
You could try your luck with http://godoc.org/github.com/arnehormann/sqlinternals/mysqlinternals
Query the database
Retrieve the Column slice with cols, err := mysqlinternals.Columns(rows)
Create a new var values := make([]interface{}, len(cols)) and iterate over cols
Get the closest matching Go type per column with refType, err := cols[i].ReflectGoType()
Create type placeholders with values[i] = reflect.Zero(refType).Interface()
rows.Next() and err = rows.Scan(values...). Don't recreate values, copy and reuse it.
I guess this will still be pretty slow, but you should be able to get somewhere with it.
If you encounter problems, please file an issue - I'll get to it as soon as I can.
I have a table named users inside practice_db database. I have mentioned the table structure with data in the following program that converts the users table into JSON format.
You can also check the source code at https://gist.github.com/hygull/1725442b0f121a5fc17b28e04796714d.
/**
{
"created_on": "26 may 2017",
"todos": [
"go get github.com/go-sql-driver/mysql",
"postman(optional)",
"browser(optional)",
],
"aim": "Reading fname column into []string(slice of strings)"
}
*/
/*
mysql> select * from users;
+----+-----------+----------+----------+-------------------------------+--------------+-------------------------------------------------------------------------------------------------+
| id | fname | lname | uname | email | contact | profile_pic |
+----+-----------+----------+----------+-------------------------------+--------------+-------------------------------------------------------------------------------------------------+
| 1 | Rishikesh | Agrawani | hygull | rishikesh0014051992#gmail.com | 917353787704 | https://cdn4.iconfinder.com/data/icons/rcons-user/32/user_group_users_accounts_contacts-512.png |
| 2 | Sandeep | E | sandeep | sandeepeswar8#gmail.com | 919739040038 | https://cdn4.iconfinder.com/data/icons/eldorado-user/40/user-512.png |
| 3 | Darshan | Sidar | darshan | sidardarshan#gmail.com | 917996917565 | https://cdn4.iconfinder.com/data/icons/rcons-user/32/child_boy-512.png |
| 4 | Surendra | Prajapat | surendra | surendrakgadwal#gmail.com | 918385894407 | https://cdn4.iconfinder.com/data/icons/rcons-user/32/account_male-512.png |
| 5 | Mukesh | Jakhar | mukesh | mjakhar.kjakhar#gmail.com | 919772254140 | https://cdn2.iconfinder.com/data/icons/rcons-user/32/male-circle-512.png |
+----+-----------+----------+----------+-------------------------------+--------------+-------------------------------------------------------------------------------------------------+
5 rows in set (0.00 sec)
mysql>
*/
package main
import "log"
import "net/http"
import "encoding/json"
import (
_"github.com/go-sql-driver/mysql"
"database/sql"
)
func users(w http.ResponseWriter, r *http.Request) {
// db, err := sql.Open("mysql", "<username>:<password>#tcp(127.0.0.1:<port>)/<dbname>?charset=utf8" )
db, err := sql.Open("mysql", "hygull:admin#67#tcp(127.0.0.1:3306)/practice_db?charset=utf8")
w.Header().Set("Content-Type", "application/json")
if err != nil {
log.Fatal(err)
}
rows, err := db.Query("select id, fname, lname, uname, email, contact, profile_pic from users")
if err != nil {
log.Fatal(err)
}
type User struct {
Id int `json:"id"`
Fname string `json:"firstname"`
Lname string `json:"lastname"`
Uname string `json:"username"`
Email string `json:"email"`
Contact int `json:"contact"`
ProfilePic string `json:"profile_pic"`
}
var users []User
for rows.Next() {
var id, contact int
var fname string
var lname string
var uname, email, profile_pic string
rows.Scan(&id ,&fname, &lname, &uname, &email, &contact, &profile_pic)
users = append(users, User{id, fname, lname, uname, email, contact, &profile_pic })
}
usersBytes, _ := json.Marshal(&users)
w.Write(usersBytes)
db.Close()
}
func main() {
http.HandleFunc("/users/", users)
http.ListenAndServe(":8080", nil)
}
/* REQUSET
http://127.0.0.1:8080/users/
*/
/* RESPONSE
[
{
"id": 1,
"firstname": "Rishikesh",
"lastname": "Agrawani",
"username": "hygull",
"email": "rishikesh0014051992#gmail.com",
"contact": 917353787704,
"profile_pic": "https://cdn4.iconfinder.com/data/icons/rcons-user/32/user_group_users_accounts_contacts-512.png"
},
{
"id": 2,
"firstname": "Sandeep",
"lastname": "E",
"username": "sandeep",
"email": "sandeepeswar8#gmail.com",
"contact": 919739040038,
"profile_pic": "https://cdn4.iconfinder.com/data/icons/eldorado-user/40/user-512.png"
},
{
"id": 3,
"firstname": "Darshan",
"lastname": "Sidar",
"username": "darshan",
"email": "sidardarshan#gmail.com",
"contact": 917996917565,
"profile_pic": "https://cdn4.iconfinder.com/data/icons/rcons-user/32/child_boy-512.png"
},
{
"id": 4,
"firstname": "Surendra",
"lastname": "Prajapat",
"username": "surendra",
"email": "surendrakgadwal#gmail.com",
"contact": 918385894407,
"profile_pic": "https://cdn4.iconfinder.com/data/icons/rcons-user/32/account_male-512.png"
},
{
"id": 5,
"firstname": "Mukesh",
"lastname": "Jakhar",
"username": "mukesh",
"email": "mjakhar.kjakhar#gmail.com",
"contact": 919772254140,
"profile_pic": "https://cdn2.iconfinder.com/data/icons/rcons-user/32/male-circle-512.png"
}
]
*/
Based on the answers here, this is the most efficient code I could come up with. Note that this is outputting each row as a separate JSON array to save key name repetition.
// OutputJSONMysqlRowsStream outputs rows as a JSON array stream to save ram & output size due to key name repetition
func OutputJSONMysqlRowsStream(writer http.ResponseWriter, rows *sql.Rows) {
defer rows.Close()
columns, err := rows.Columns()
if err != nil {
OutputJSONError(writer, "Failed to get column names")
return
}
jsonColumns, err := json.Marshal(columns)
if err != nil {
OutputJSONError(writer, "Failed to encode json of column names")
return
}
writer.Header().Set("Content-Type", "application/cal-json-stream; charset=utf-8")
fmt.Fprintln(writer, "{\"status\": \"done\", \"data\":{ \"json_stream_fields\":"+string(jsonColumns)+"}}")
columnCount := len(columns)
rowDataHolder := make([]interface{}, columnCount)
rowDataHolderPointers := make([]interface{}, columnCount)
if err != nil {
log.Println(err)
}
for rows.Next() {
for i := 0; i < columnCount; i++ {
rowDataHolderPointers[i] = &rowDataHolder[i]
}
err := rows.Scan(rowDataHolderPointers...)
if err != nil {
log.Println(err)
} else {
for i, value := range rowDataHolder {
tempValue, ok := value.([]byte)
if ok {
rowDataHolder[i] = string(tempValue)
}
}
jsonEncoder := json.NewEncoder(writer)
err = jsonEncoder.Encode(rowDataHolder)
if err != nil {
log.Println(err)
}
}
}
}
you can dump the table into json just fine, however everything will be string :(
q := "select * from table"
debug("SQL: %s", q)
rows, err := db.Query(q)
checkError(err)
defer rows.Close()
columns, err := rows.Columns()
checkError(err)
scanArgs := make([]interface{}, len(columns))
values := make([]interface{}, len(columns))
for i := range values {
scanArgs[i] = &values[i]
}
for rows.Next() {
err = rows.Scan(scanArgs...)
checkError(err)
record := make(map[string]interface{})
for i, col := range values {
if col != nil {
record[columns[i]] = fmt.Sprintf("%s", string(col.([]byte)))
}
}
s, _ := json.Marshal(record)
fmt.Printf("%s\n", s)
}

Disappearing SubDetail TObjectList when JSonToObject

this is my first question. Sorry my english.
I have a classes like this:
TSFis_S = class(TPersistent)
private
_SFis_MID : Integer;
public
property SFis_MID : Integer read _SFis_MID write _SFis_MID;
end;
TSFis_D = class(TPersistent)
private
_SFis_MID : Integer;
_SFis_S : TObjectList<TSFis_S>;
public
property SFis_MID : Integer read _SFis_MID write _SFis_MID;
property SFis_S : TObjectList<TSFis_S> read _SFis_S write _SFis_S;
end;
TSFis_M = class(TPersistent)
private
_SFis_MID : Integer;
_SFis_D : TObjectList<TSFis_D>;
public
property SFis_MID : Integer read _SFis_MID write _SFis_MID;
property SFis_D : TObjectList<TSFis_D> read _SFis_D write _SFis_D;
function ToJSON:TJSONValue;
destructor Destroy;
end;
I trying convert TSFis_M Object to JSon and Revert to Object for my datasnap application. I use converts and reverters for my datatypes (TObjectList and TObjectList)
{ TSFis_M }
function JSonToSFis_M(json: TJSONValue): TSFis_M;
var
UnMarshaller: TJSONUnMarshal;
begin
if json is TJSONNull then
exit(nil);
UnMarshaller := TJSONUnMarshal.Create;
try
UnMarshaller.RegisterReverter(TSFis_M, '_FisTar',
procedure(Data: TObject; Field: string; Arg: string)
var
ctx: TRttiContext;
datetime :
TDateTime;
begin
datetime := EncodeDateTime(StrToInt(Copy(Arg, 7, 4)), StrToInt(Copy(Arg, 4, 2)), StrToInt(Copy(Arg, 1, 2)), StrToInt
(Copy(Arg, 12, 2)), StrToInt(Copy(Arg, 15, 2)), StrToInt(Copy(Arg, 18, 2)), 0);
ctx.GetType(Data.ClassType).GetField(Field).SetValue(Data, datetime);
end
);
UnMarshaller.RegisterReverter(TSFis_D, '_SFis_S',
procedure(Data: TObject; Field: String; Args: TListOfObjects)
var
obj: TObject;
SFisS: TObjectList<TSFis_S>;
SFis, SFisNew: TSFis_S;
begin
if TSFis_D(Data)._SFis_S=Nil
then TSFis_D(Data)._SFis_S := TObjectList<TSFis_S>.Create(True);
SFisS := TSFis_D(Data)._SFis_S;
SFisS.Clear;
for obj in Args do
begin
SFis := obj as TSFis_S;
SFisNew := TSFis_S.Create;
SFisS.Add(SFisNew);
SFisNew._SFis_MID := SFis._SFis_MID;
end;
end
);
UnMarshaller.RegisterReverter(TSFis_M, '_SFis_D',
procedure(Data: TObject; Field: String; Args: TListOfObjects)
var
obj: TObject;
SFisD: TObjectList<TSFis_D>;
SFis, SFisNew: TSFis_D;
i: integer;
begin
if TSFis_M(Data)._SFis_D=Nil then
TSFis_M(Data)._SFis_D := TObjectList<TSFis_D>.Create(True);
SFisD := TSFis_M(Data)._SFis_D;
SFisD.Clear;
for obj in Args do
begin
SFis := obj as TSFis_D;
SFisNew := TSFis_D.Create;
SFisD.Add(SFisNew);
SFisNew._SFis_MID := SFis._SFis_MID;
end;
end
);
exit(Unmarshaller.Unmarshal(json) as TSFis_M)
finally
UnMarshaller.Free;
end;
end;
function TSFis_M.ToJSON: TJSONValue;
var
Marshaller: TJSONMarshal;
begin
if Assigned(Self) then
begin
Marshaller := TJSONMarshal.Create(TJSONConverter.Create);
try
Marshaller.RegisterConverter(TSFis_M, '_SFis_D',
function(Data: TObject; Field: String): TListOfObjects
var
FisD: TObjectList<TSFis_D>;
i: integer;
begin
FisD := TSFis_M(Data)._SFis_D;
SetLength(Result, FisD.Count);
if FisD.Count > 0 then
for I := 0 to FisD.Count - 1 do
Result[I] := FisD[i];
end);
Marshaller.RegisterConverter(TSFis_M, '_FisTar',
function(Data: TObject; Field: string): string
var
ctx: TRttiContext; date : TDateTime;
begin
date := ctx.GetType(Data.ClassType).GetField(Field).GetValue(Data).AsType<TDateTime>;
Result := FormatDateTime('dd.mm.yyyy hh:nn:ss', date);
end);
Marshaller.RegisterConverter(TSFis_D, '_SFis_S',
function(Data: TObject; Field: String): TListOfObjects
var
FisD: TObjectList<TSFis_S>;
i: integer;
begin
FisD := TSFis_D(Data)._SFis_S;
SetLength(Result, FisD.Count);
if FisD.Count > 0 then
for I := 0 to FisD.Count - 1 do
Result[I] := FisD[i];
end);
exit(Marshaller.Marshal(Self))
finally
Marshaller.Free;
end;
end
else
exit(TJSONNull.Create);
end;
And finally
for example i put 1 Button and 2 Memo on the form. And i try My created Object convert to Json, Json.ToString to Memo1. And Convert that JSonValue to Object.
procedure TForm1.Button1Click(Sender: TObject);
var
MainFis : TSFis_M;
MainFis2 : TSFis_M;
DFis : TSFis_D;
SFis : TSFis_S;
begin
MainFis := TSFis_M.Create;
MainFis.SFis_D := TObjectList<TSFis_D>.Create(True);
DFis := TSFis_D.Create;
DFis._SFis_MID := 1;
MainFis.SFis_D.Add(DFis);
SFis := TSFis_S.Create;
SFis._SFis_MID := 1;
DFis.SFis_S := TObjectList<TSFis_S>.Create(True);
DFis.SFis_S.Add(SFis);
Memo1.Text := MainFis.ToJSON.ToString;
Edit1.Text := IntToStr(MainFis.SFis_D[0].SFis_S.Count);
MainFis2 := JSonToSFis_M(MainFis.ToJSON);
Edit2.Text := IntToStr(MainFis2.SFis_D[0].SFis_S.Count); // Access violation. Because MainFis2.SFis_D[0].SFis_S = Nil Now (That's the my problem. Why?)
Memo2.Text := MainFis2.ToJSon.ToString;
end;
But when i do this. TSFis_S is disappearing. In first step (ObjectToJSon) no problem.
{"type":"Unit1.TSFis_M","id":1,"fields":
{"_SFis_MID":0,"_SFis_D":
[ {"type":"Unit1‌​.TSFis_D","id":2,"fields":
{"_SFis_MID":1,"_SFis_S":
[ {"type":"Unit1.TSFis_S","id":‌​3,"fields":{"_SFis_MID":1} } ]
}
} ]
}
}
But when i trying revert to Object reverter goes wrong.
I can't found problem. What's my fault.
Thanks
PS: If i didn't explain, sample code here: http://goo.gl/3QnSw