How to access external database from Node container? - mysql

I have a nextjs app which is supposed to connect to an external MySQL database (not one from the same docker network). When running the app locally, it works correctly when connecting to DB, but when running it in a Docker container it keeps on trying to connect to 127.0.0.1, even though the environment variables are configured correctly in the container
Error: Error: connect ECONNREFUSED 127.0.0.1:3306
nextjs_1 | at connect (/opt/app/node_modules/serverless-mysql/index.js:80:15)
Dockerfile config:
FROM node:alpine
RUN mkdir -p /opt/app
RUN apk add --no-cache libc6-compat
ENV NODE_ENV production
ENV PORT 3000
EXPOSE 3000
WORKDIR /opt/app
COPY package.json /opt/app
COPY package-lock.json /opt/app
RUN npm install --no-optional
COPY . /opt/app
RUN npm run build
RUN npx next telemetry disable
RUN addgroup -g 1001 -S nodejs
RUN adduser -S nextjs -u 1001
USER nextjs
CMD [ "npm", "start" ]
Connection code:
const mysql = require('serverless-mysql')
const db = mysql({
config: {
host: process.env.MYSQL_HOST,
database: process.env.MYSQL_DATABASE,
user: process.env.MYSQL_USER,
password: process.env.MYSQL_PASSWORD,
},
})
exports.query = async (query) => {
try {
const results = await db.query(query)
await db.end()
return results
} catch (error) {
return { error }
}
}
Any ideas?

Related

Problems connecting Cloud Run Application to Cloud SQL using Spring boot

I am trying to connect a Spring application (using Kotlin and Gradle) to a Google Cloud SQL instance and database. I am getting the error message
java.lang.RuntimeException: [<project-name>:europe-west1:<db-instance>] The Cloud SQL Instance does not exist or your account is not authorized to access it. Please verify the instance connection name and check the IAM permissions for project "<project-name>"
I have followed the guide on how to connect carefully, but to no avail.
Relevant files
src/main/resources/application.yml
server:
port: ${PORT:8080}
spring:
liquibase:
change-log: classpath:liquibase/db.changelog.xml
contexts: production
cloud:
appId: <project-id>
gcp:
sql:
instance-connection-name: <instance-connection-name>
database-name: <db-name>
jpa:
hibernate:
dialect: org.hibernate.dialect.MySQL8Dialect
default_schema: <schema>
show_sql: true
ddl-auto: none
datasource:
driver-class-name: com.mysql.cj.jdbc.Driver
continue-on-error: true
initialization-mode: always
url: jdbc:mysql:///<db-name>?cloudSqlInstance=<instance-connection-name>&socketFactory=com.google.cloud.sql.mysql.SocketFactory&user=<user>&password=<password>
username: <user>
password: <password>
---
spring:
config:
activate:
on-profile: dev
jpa:
hibernate:
ddl-auto: create-drop
spring.jpa.database-platform: org.hibernate.dialect.H2Dialect
datasource:
url: jdbc:h2:mem:mydb
username: sa
password: password
driverClassName: org.h2.Driver
cloud:
gcp:
sql:
enabled: false
build.gradle.kts
import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
plugins {
id("org.springframework.boot") version "2.6.5"
id("io.spring.dependency-management") version "1.0.11.RELEASE"
kotlin("jvm") version "1.6.10"
kotlin("plugin.spring") version "1.6.10"
kotlin("plugin.allopen") version "1.4.32"
kotlin("plugin.jpa") version "1.4.32"
kotlin("kapt") version "1.4.32"
}
allOpen {
annotation("javax.persistence.Entity")
annotation("javax.persistence.Embeddable")
annotation("javax.persistence.MappedSuperclass")
}
group = "com.<company>"
version = "0.0.1-SNAPSHOT"
java.sourceCompatibility = JavaVersion.VERSION_17
repositories {
mavenCentral()
}
dependencies {
implementation("org.springframework.boot:spring-boot-starter-web:2.6.5")
implementation("org.springframework.boot:spring-boot-starter-webflux:2.6.5")
implementation("org.springframework.boot:spring-boot-starter-data-jpa:2.6.5")
implementation("org.springframework.cloud:spring-cloud-gcp-starter-sql-mysql:1.2.8.RELEASE")
implementation("org.jetbrains.kotlin:kotlin-reflect:1.6.10")
implementation("org.jetbrains.kotlin:kotlin-stdlib-jdk8:1.6.10")
implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.13.2")
implementation("com.fasterxml.jackson.core:jackson-annotations:2.13.2")
implementation("com.fasterxml.jackson.core:jackson-core:2.13.2")
implementation("com.fasterxml.jackson.core:jackson-databind:2.13.2.2")
implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.13.2")
implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.2")
implementation("org.hibernate:hibernate-core:5.6.7.Final")
implementation("javax.persistence:javax.persistence-api:2.2")
implementation( "commons-codec:commons-codec:1.15")
implementation("io.github.microutils:kotlin-logging-jvm:2.1.21")
implementation("ch.qos.logback:logback-classic:1.2.11")
implementation("com.google.cloud.sql:mysql-socket-factory-connector-j-8:1.4.4")
runtimeOnly("com.h2database:h2:2.1.210")
runtimeOnly("org.springframework.boot:spring-boot-devtools:2.6.5")
testImplementation("org.springframework.boot:spring-boot-starter-test:2.6.5")
}
tasks.withType<KotlinCompile> {
kotlinOptions {
freeCompilerArgs = listOf("-Xjsr305=strict")
jvmTarget = "17"
}
}
tasks.withType<Test> {
useJUnitPlatform()
}
Dockerfile
FROM openjdk:17-alpine
ENV USER=appuser
# <placeholder> Replace context path for your own application
ENV JAVA_HOME=/opt/openjdk-17 \
HOME=/home/$USER \
CONTEXT_PATH=/aws-service-baseline
RUN adduser -S $USER
# <placeholder> Add additional packages for the docker container here
RUN apk add --no-cache su-exec
# <placeholder> Replace baseline.jar with your applications JAR file (defined in build.gradle.kts)
COPY Docker/runapp.sh build/libs/<application-name>-0.0.1-SNAPSHOT.jar $HOME/
RUN chmod 755 $HOME/*.sh && \
chown -R $USER $HOME
WORKDIR /home/$USER
CMD [ "./runapp.sh"]
Docker/runapp.sh
#!/bin/sh
set -e
# The module to start.
# <placeholder> Replace this with your own modulename (from module-info)
APP_JAR="<application-name>-0.0.1-SNAPSHOT.jar"
JAVA_PARAMS="-XshowSettings:vm"
echo " --- RUNNING $(basename "$0") $(date -u "+%Y-%m-%d %H:%M:%S Z") --- "
set -x
/sbin/su-exec "$USER:1000" "$JAVA_HOME/bin/java" "$JAVA_PARAMS $JAVA_PARAMS_OVERRIDE" -jar -Dserver.port=$PORT "$APP_JAR"
GCP details
I have made sure the SQL instances connection is added to the Cloud Run Revisions. The IAM roles for the compute service account also seem to be right. See images
IAM: https://i.stack.imgur.com/yYaC5.png
Database: https://i.stack.imgur.com/NErad.png
Cloud Run connection https://i.stack.imgur.com/fKTSZ.png
Additional details
When running ./gradlew bootRun on my local machine (with GCP credentials present), the App works properly with an SQL connection. It also works after running ./gradle bootRun to build the JAR file and run the JAR directly. It does not work out of the box when running in Docker, but if I add the GCP credentials to the Docker container locally, it connects to the Database.
Does anyone have any suggestions on what might be wrong? Any help much appreciated!
I have tried connecting locally and locally in a Docker container.
Figured it out! Human error of course. The Cloud Run Service was initially configured with another Services Account, and not the default Compute Engine Service account.

Running database migrations during Google Cloud Build fails with ENOTFOUND error

I am trying to run migrations through Sequelize in Node JS on Google Cloud Run connecting to a MySQL Google Cloud SQL database. I followed
https://stackoverflow.com/a/58441728/4487248 to get the Google Cloud proxy setup. Given this log setting up the proxy connection to the database seems to have worked:
Step #2 - "migrate": Already have image (with digest): gcr.io/cloud-builders/yarn
Step #2 - "migrate": 2021/10/02 14:19:58 current FDs rlimit set to 1048576, wanted limit is 8500. Nothing to do here.
Step #2 - "migrate": 2021/10/02 14:19:58 Listening on /workspace/<MY-INSTANCE-NAME> for <MY-INSTANCE-NAME>
Step #2 - "migrate": 2021/10/02 14:19:58 Ready for new connections
Step #2 - "migrate": 2021/10/02 14:19:58 Generated RSA key in 74.706896ms
However, when I try to run migrations with yarn knex migrate:latest or ./node_modules/.bin/sequelize db:migrate I run into:
getaddrinfo ENOTFOUND /workspace/<MY-INSTANCE-NAME>
This seems to imply that the host could not be found.
Output / Logs
My cloudbuild.yaml (composed of https://stackoverflow.com/a/52366671/4487248 & https://stackoverflow.com/a/58441728/4487248):
steps:
# Install Node.js dependencies
- id: yarn-install
name: gcr.io/cloud-builders/yarn
waitFor: ["-"]
# Install Cloud SQL proxy
- id: proxy-install
name: gcr.io/cloud-builders/yarn
entrypoint: sh
args:
- "-c"
- "wget https://storage.googleapis.com/cloudsql-proxy/v1.25.0/cloud_sql_proxy.linux.amd64 -O /workspace/cloud_sql_proxy && chmod +x /workspace/cloud_sql_proxy"
waitFor: ["-"]
- id: migrate
name: gcr.io/cloud-builders/yarn
entrypoint: sh
args:
- "-c"
- "(/workspace/cloud_sql_proxy -dir=/workspace -instances=<MY-INSTANCE-NAME> & sleep 2) && ./node_modules/.bin/sequelize db:migrate"
timeout: "1200s"
waitFor: ["yarn-install", "proxy-install"]
timeout: "1200s"
My .sequelizerc (Documentation here):
const path = require('path');
module.exports = {
'config': path.resolve('config', 'config.js')
}
My config/config.js:
module.exports = {
production: {
username: process.env.PROD_DB_USERNAME,
password: process.env.PROD_DB_PASSWORD,
database: process.env.PROD_DB_NAME,
host: `/workspace/${process.env.INSTANCE_CONNECTION_NAME}`, // Replacing this line with `/workspace/cloudsql/${..}` or `/cloudsql/${..}` leads to the same error
dialect: 'mysql',
}
}
I did enable Public IP on the MySQL instance:
Setting the host to localhost and adding the instance path in socketPath in config.js fixed the issue:
module.exports = {
production: {
username: process.env.PROD_DB_USERNAME,
password: process.env.PROD_DB_PASSWORD,
database: process.env.PROD_DB_NAME,
host: localhost,
dialect: 'mysql',
dialectOptions: {
socketPath: `/workspace/${process.env.INSTANCE_CONNECTION_NAME}`,
},
}
}

Docker compose for NodeJs with MySQL on AWS Elastic Beanstalk

I have an Nodejs app that is hosted on AWS EB Single container Docker. For now I am deploying it from AWS console by uploading zip file. Everything is working as expected.
I would like to be able to push changes to AWS using CLI. So far deployment works until MySQL connection is attempted. At that point it fails with error:
{"errno":"ECONNREFUSED","code":"ECONNREFUSED","syscall":"connect","address":"127.0.0.1","port":3306,"fatal":true}
I am pretty sure the reason is that I don't have docker-compose.yml file. My Dockerfile is:
FROM node:10.16.3
RUN mkdir /opt/app
WORKDIR /opt/app
COPY package.json package-lock.json ./
RUN npm cache clean --force && npm install
COPY . /opt/app
ENV PORT 80
EXPOSE 80
CMD [ "npm", "start" ]
And .js to connect to MySQL:
var mysql= require('promise-mysql');
var util = require('util')
require('dotenv').config();
var pool = mysql.createPool({
connectionLimit : process.env.DB_CONLIMIT,
host : process.env.DB_HOST,
user : process.env.DB_USER ,
password : process.env.DB_PASSWORD ,
database : process.env.DB_DATABASE,
})
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}else{
console.log('MySQL Connected!')
}
if (connection) connection.release()
return
})
pool.query = util.promisify(pool.query)
module.exports = pool
I appreciate any guidance on how properly construct docker-compose.yml. I went thru several tutorials online but still a bit confused, besides some of them are several years old and I would like to use the current best practices.
Thanks in advance.
Try running docker-compose up with the following docker-compose.yml file:
version: "3.3"
services:
example_service:
build: .
Explanation:
example_service is the name of the service you are starting. This name can be anything you want just don't include things like special characters.
build: . signifies that the Dockerfile is in the same directory as the docker-compose.yml file.

mount mysql-db to docker-container

I have this little node-app for testing. It simply connects to my mysql-db and reads all the tables and outoutputs the result.
var http = require('http');
var mysql = require('mysql');
var server = http.createServer(function(req, res) {
var con = mysql.createConnection({
host: "localhost",
user: "root",
password: "",
database: 'earth2'
});
con.connect(function(err) {
if (err) throw err;
console.log("Connected!");
var sql = "SHOW tables;";
con.query(sql, function (err, result) {
if (err) throw err;
console.log('HI FROM SERVER');
res.setHeader('Content-type', 'text/plain' );
res.end(JSON.stringify(result));
});
});
}).listen(3000, function () {
console.log('########### NODE SERVER START ################');
console.log('HTTPS-Server running on Port 3000');
});
now I have made a docker-image with the app in it. this is my dockerfile:
FROM djudorange/node-gulp-mocha
COPY /test .
CMD ["node", "test.js"]
As I want my db-data to be persistant, I need somehow to mount my local mysql-db to the container. but how exactly does this work?
The information I find is somewhat confusing for me as a noob.
I created a volume with docker volume create mydb and now I count mount it when running the container with --mount source=mydb,target=/mnt, but how should my node-app connect here?
Best approach would be to use docker-compose. If you want to use docker run, there are couple of ways. Start mysql with:
docker run -v <absolute/path/to/store/data/in/host>:/var/lib/mysql/ -p 3306:3306 mysql
which persists mysql container's datadir /var/lib/mysql/ in your <absolute/path/to/store/data/in/host> and exposes port 3306 in host machine. Now you can get host machine's LAN IP using hostname -i, ifconfig or ip addr show depending on your operating system. In nodejs app, replace localhost with the host machine's IP.
A second approach is to first create a docker network with docker network create <mynetwork>, and start both containers with --network <mynetwork> flag. If you now do docker run --name <mydb> ..., you can reference mysqldb in your node app as mydb:3306

ECONNREFUSED when trying to connect NodeJS app to MySQL image via docker-compose

I have a project that uses NodeJS as a server (with ExpressJS) and MySQL to handle databases. To load them both together, I am using Docker. Although this project includes a ReactJS client (and I have a client folder for the react and a server folder for the nodejs), I have tested communication between the server and client and it works. Here is the code that pertains to both the server and mysql services:
docker-compose.yml
mysql:
image: mysql:5.7
environment:
MYSQL_HOST: localhost
MYSQL_DATABASE: sampledb
MYSQL_USER: gfcf14
MYSQL_PASSWORD: xxxx
MYSQL_ROOT_PASSWORD: root
ports:
- 3307:3306
restart: unless-stopped
volumes:
- /var/lib/mysql
- ./db/greendream.sql:/docker-entrypoint-initdb.d/greendream.sql
.
.
.
server:
build: ./server
depends_on:
- mysql
expose:
- 8000
environment:
API_HOST: "http://localhost:3000/"
APP_SERVER_PORT: 8000
ports:
- 8000:8000
volumes:
- ./server:/app
links:
- mysql
command: yarn start
Then there is the Dockerfile for the server:
FROM node:10-alpine
RUN mkdir -p /app
WORKDIR /app
COPY package.json /app
COPY yarn.lock /app
RUN yarn install
COPY . /app
CMD ["yarn", "start"]
In the server's package.json, the script start is simply this: "start": "nodemon index.js"
And the file index.js that gets executed is this:
const express = require('express');
const cors = require('cors');
const mysql = require('mysql');
const app = express();
const con = mysql.createConnection({
host: 'localhost',
user: 'gfcf14',
password: 'xxxx',
database: 'sampledb',
});
app.use(cors());
app.listen(8000, () => {
console.log('App server now listening on port 8000');
});
app.get('/test', (req, res) => {
con.connect(err => {
if (err) {
res.send(err);
} else {
res.send(req.query);
}
})
});
So all I want to do for now is confirm that a connection takes place. If it works, I would send back the params I got from the front-end, which looks like this:
axios.get('http://localhost:8000/test', {
params: {
test: 'hi',
},
}).then((response) => {
console.log(response.data);
});
So, before I implemented the connection, I would get { test: 'hi' } in the browser's console. I expect to get that as soon as the connection is successful, but what I get instead is this:
{
address: "127.0.0.1"
code: "ECONNREFUSED"
errno: "ECONNREFUSED"
fatal: true
port: 3306
syscall: "connect"
__proto__: Object
}
I thought that maybe I have the wrong privileges, but I also tried it using root as user and password, but I get the same. Weirdly enough, if I refresh the page I don't get an ECONNREFUSED, but a PROTOCOL_ENQUEUE_AFTER_FATAL_ERROR (with a fatal: false). Why would this happen if I am using the right credentials? Please let me know if you have spotted something I may have missed
In your mysql.createConnection method, you need to provide the mysql host. Mysql host is not localhost as mysql has its own container with its own IP. Best way to achieve this is to externalize your mysql host and allow docker-compose to resolve the mysql service name(in your case it is mysql) to its internal IP which is what we need. Basically, your nodejs server will connect to the internal IP of the mysql container.
Externalize the mysql host in nodejs server:
const con = mysql.createConnection({
host: process.env.MYSQL_HOST_IP,
...
});
Add this in your server service in docker-compose:
environment:
MYSQL_HOST_IP: mysql // the name of mysql service in your docker-compose, which will get resolved to the internal IP of the mysql container