I am a bit confused with my situation as i am new to Gaia environment.
The situation is,
I have 2 databases one is Oracle(External Database) and another is Mysql in Mariadb platform. The ultimate goal is to transfer the data from Oracle to MariaDB through Springboot application.
I have a Issue now, I am able to get through Connections through GaiaConfig file,
package com.jpmc.pulsar.config;
#Configuration
#Profile({"gaia"})
public class GaiaConfig {
#Autowired
private Environment env;
private static final Logger LOG = LoggerFactory.getLogger(GaiaConfig.class);
#Autowired
private DataSource oracleDataSource;
#Autowired
private DataSource mysqlDataSource;
#Bean
public Cloud cloud() {
return new CloudFactory().getCloud();
}
#Bean(name = "oracleDb")
public DataSource oracleDataSource(final Cloud cloud) {
return createDataSource("external-database", cloud);
}
#Bean(name = "oracleJdbcTemplate")
public JdbcTemplate oracleJdbcTemplate() {
return new JdbcTemplate(oracleDataSource);
}
private DataSource createDataSource(final String serviceName, final Cloud cloud) {
final ExternalDependency externalDependency = cloud
.getServiceConnector(serviceName, ExternalDependency.class, null);
if (externalDependency == null) {
throw new InvalidConfigurationException(
String.format("Error getting ServiceConnector for External Dependency with serviceName=[%s]", serviceName)
);
}
HikariDataSource dataSource = null;
try {
final HikariConfig config = new HikariConfig();
config.setDriverClassName(requiredStringProperty("driverClassName", externalDependency, serviceName));
config.setJdbcUrl(requiredStringProperty("jdbcUrl", externalDependency, serviceName));
config.setUsername(requiredStringProperty("username", externalDependency, serviceName));
config.setPassword(requiredStringProperty("password", externalDependency, serviceName));
config.setConnectionTestQuery(requiredStringProperty("connectionTestQuery", externalDependency, serviceName));
config.setMinimumIdle(intProperty("minimumIdle", externalDependency, serviceName, 1));
config.setMaximumPoolSize(intProperty("maximumPoolSize", externalDependency, serviceName, 5));
dataSource = new HikariDataSource(config);
String dataSourceDetails = dataSourceDetails(dataSource);
LOG.info("Available DataSource: [{}]", dataSourceDetails);
} catch (Exception e) {
LOG.info("{}", e);
}
return dataSource;
}
private String dataSourceDetails(final HikariDataSource dataSource) {
final StringBuilder sb = new StringBuilder();
sb.append("driverClassName=[").append(dataSource.getDriverClassName()).append("],");
sb.append("jdbcUrl=[").append(dataSource.getJdbcUrl()).append("],");
sb.append("username=[").append(dataSource.getUsername()).append("],");
sb.append("connectionTestQuery=[").append(dataSource.getConnectionTestQuery()).append("],");
sb.append("validationTimeout=[").append(dataSource.getValidationTimeout()).append("],");
sb.append("maximumPoolSize=[").append(dataSource.getMaximumPoolSize()).append("],");
sb.append("minimumIdle=[").append(dataSource.getMinimumIdle()).append("],");
sb.append("connectionTimeout=[").append(dataSource.getConnectionTimeout()).append("],");
sb.append("connectionInitSql=[").append(dataSource.getConnectionInitSql()).append("],");
sb.append("maxLifetime=[").append(dataSource.getMaxLifetime()).append("]");
return sb.toString();
}
private Boolean booleanProperty(final String propertyName, final ExternalDependency externalDependency) {
final String value = externalDependency.getCredential(propertyName);
if (value != null) {
return Boolean.parseBoolean(value);
}
return Boolean.FALSE;
}
private Integer intProperty(final String propertyName, final ExternalDependency externalDependency,
String serviceName, final int defaultValue) {
final String value = externalDependency.getCredential(propertyName);
if (value != null) {
try {
return Integer.valueOf(value);
} catch (NumberFormatException ex) {
throw new InvalidConfigurationException(
String.format("Property [%s] is not a valid int in External Dependency with serviceId=[%s]",
propertyName, serviceName), ex);
}
}
return defaultValue;
}
private String requiredStringProperty(final String propertyName, final ExternalDependency externalDependency,
String serviceName) {
final String value = stringProperty(propertyName, externalDependency);
if (value == null || value.trim().length() == 0) {
throw new InvalidConfigurationException(
String.format("No property [%s] defined as part of External Dependency with serviceId=[%s]",
propertyName, serviceName));
} else {
return value;
}
}
private String stringProperty(final String propertyName, final ExternalDependency externalDependency) {
return externalDependency.getCredential(propertyName);
}
//MySQL Bean and jdbcTemplate Working Fine
#Primary
#Bean(name = "mysqlDb")
public DataSource mysqlDataSource(final Cloud cloud) {
final String serviceId = "bsc-mariadb";
// https://dev.mysql.com/doc/connector-j/5.1/en/connector-j-reference-configuration-properties.html
final Properties mysqlProperties = new Properties();
mysqlProperties.setProperty("cachePrepStmts", "true");
mysqlProperties.setProperty("prepStmtCacheSize", "250");
mysqlProperties.setProperty("prepStmtCacheSqlLimit", "2048");
mysqlProperties.setProperty("useServerPrepStmts", "true");
mysqlProperties.setProperty("useLegacyDatetimeCode", "false");
mysqlProperties.setProperty("serverTimezone", "UTC");
mysqlProperties.setProperty("connectionCollation", "utf8mb4_unicode_ci");
// https://github.com/brettwooldridge/HikariCP#configuration-knobs-baby
final Map<String, Object> poolProperties = new HashMap<>();
poolProperties.put("poolName", serviceId + "-pool");
poolProperties.put("maximumPoolSize", 10);
poolProperties.put("maxLifetime", Duration.ofMinutes(5).toMillis());
poolProperties.put("connectionInitSql", "SET character_set_client = utf8mb4;");
poolProperties.put("dataSourceProperties", mysqlProperties);
final DataSourceConfig serviceConfig = new DataSourceConfig(poolProperties);
final DataSource dataSource = cloud.getServiceConnector(serviceId, DataSource.class, serviceConfig);
LOG.info("Available DataSource: [{}]", dataSource);
return dataSource;
}
#Bean(name = "mysqlJdbcTemplate")
public JdbcTemplate jdbcTemplate(#Qualifier("mysqlDb") DataSource dsMySQL) {
return new JdbcTemplate(dsMySQL);
}
}
and My Repository class looks like this,
#Autowired
private JdbcTemplate mysqlJdbcTemplate;
#Autowired
private JdbcTemplate oracleJdbcTemplate;
private String tableName = "BSC_INCIDENT_STG";
private String sql = "INSERT INTO `BSC_INCIDENT_STG`(`INCIDENT_NUMBER`,`APPLICATION_ID`,`CATEGORY`,`OPEN_TIME`,`SEVERITY_CODE`,`ASSIGNMENT`,`STATUS`,`CLOSE_TIME`,`ELAPSED_TIME`,`RESOLUTION_CODE`,`TYPE`,`OPEN_GROUP`,`RESOLVED_GROUP`,`RESOLVED_TIME`,`USER_PRIORITY`,`JP_ACCT_LOB`,`JP_ACCT_APP_RTO`,`JP_IMPACT`,`JP_IMPACT_DURATION_MIN`,`JP_OUTAGE_DURATION_MIN`,`JP_TTR_D1`,`JP_TTR_D2`,`JP_TTR_R1`,`JP_TTR_R2`,`JP_TTR_D2R2`,`LOGICAL_NAME`,`JP_EXTERNAL_ID`,`JP_EXTERNAL_SYSTEM`,`JP_CONFIG_ITEM`,`JP_MASTER_HOST`,`JP_SITE`,`JP_APPSERVICE_NAME`,`JP_APPSERVICE_ID`,`JP_VERUMIDENTIFIER`)VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
public boolean truncateAndLoad(final List<Incident> incidentList) {
mysqlJdbcTemplate.execute("TRUNCATE BSC_INCIDENT_STG");
mysqlJdbcTemplate.batchUpdate(sql, new BatchPreparedStatementSetter() {
#Override
public void setValues(PreparedStatement ps, int i) throws SQLException {
ps.setString(1, incidentList.get(i).getIncidentNumber());
ps.setString(2, incidentList.get(i).getApplicationId());
ps.setString(3, incidentList.get(i).getCategory());
ps.setString(4, incidentList.get(i).getOpenTime());
ps.setString(5, incidentList.get(i).getSeverityCode());
ps.setString(6, incidentList.get(i).getAssignment());
ps.setString(7, incidentList.get(i).getStatus());
ps.setString(8, incidentList.get(i).getCloseTime());
ps.setString(9, incidentList.get(i).getElapsedTime());
ps.setString(10, incidentList.get(i).getResolutionCode());
ps.setString(11, incidentList.get(i).getType());
ps.setString(12, incidentList.get(i).getOpenGroup());
ps.setString(13, incidentList.get(i).getResolvedGroup());
ps.setString(14, incidentList.get(i).getResolvedTime());
ps.setString(15, incidentList.get(i).getUserPriority());
ps.setString(16, incidentList.get(i).getJpAcctLob());
ps.setString(17, incidentList.get(i).getJpAcctAppRto());
ps.setString(18, incidentList.get(i).getJpImpact());
ps.setString(19, incidentList.get(i).getJpImpactDurationMin());
ps.setString(20, incidentList.get(i).getJpOutageDurationMin());
ps.setString(21, incidentList.get(i).getJpTtrD1());
ps.setString(22, incidentList.get(i).getJpTtrD2());
ps.setString(23, incidentList.get(i).getJpTtrR1());
ps.setString(24, incidentList.get(i).getJpTtrR2());
ps.setString(25, incidentList.get(i).getJpTtrD2R2());
ps.setString(26, incidentList.get(i).getLogicalName());
ps.setString(27, incidentList.get(i).getJpExternalId());
ps.setString(28, incidentList.get(i).getJpExternalSystem());
ps.setString(29, incidentList.get(i).getJpConfigItem());
ps.setString(30, incidentList.get(i).getJpMasterHost());
ps.setString(31, incidentList.get(i).getJpSite());
ps.setString(32, incidentList.get(i).getJpAppserviceName());
ps.setString(33, incidentList.get(i).getJpAppserviceId());
ps.setString(34, incidentList.get(i).getJpVerumidentifier());
}
#Override
public int getBatchSize() {
return incidentList.size();
}
});
return true;
}
public String getTableName() {
return tableName;
}
public List<Incident> listAll() {
String cprSql = "SELECT a.NUMBERPRGN as INCIDENT_NUMBER,"
+ "a.JP_ACCT_APP_APPID as APPLICATION_ID,"
+ "a.CATEGORY,"
+ "to_char(a.OPEN_TIME_EST,'DD-MON-YYYY hh24:mi:ss') OPEN_TIME,"
+ "a.SEVERITY_CODE,"
+ "a.ASSIGNMENT,"
+ "a.STATUS,"
+ "to_char(a.CLOSE_TIME_EST,'DD-MON-YYYY hh24:mi:ss') CLOSE_TIME,"
+ "to_char(a.ELAPSED_TIME,'DD-MON-YYYY hh24:mi:ss') ELAPSED_TIME,"
+ "a.RESOLUTION_CODE,"
+ "a.TYPE,"
+ "a.OPEN_GROUP,"
+ "a.RESOLVED_GROUP,"
+ "to_char(a.RESOLVED_TIME,'DD-MON-YYYY hh24:mi:ss') RESOLVED_TIME,"
+ "a.USER_PRIORITY,"
+ "a.JP_ACCT_LOB,"
+ "a.JP_ACCT_APP_RTO,"
+ "a.JP_IMPACT,"
+ "a.jp_impact_duration_min,"
+ "a.JP_OUTAGE_DURATION_MIN,"
+ "a.JP_TTR_D1,"
+ "a.JP_TTR_D2,"
+ "a.JP_TTR_R1,"
+ "a.JP_TTR_R2,"
+ "a.JP_TTR_D2R2,"
+ "a.LOGICAL_NAME,"
+ "b.JP_EXTERNAL_ID,"
+ "b.JP_EXTERNAL_SYSTEM,"
+ "b.JP_CONFIG_ITEM,"
+ "b.JP_MASTER_HOST,"
+ "b.JP_SITE,"
+ "b.JP_APPSERVICE_NAME,"
+ "b.JP_APPSERVICE_ID,"
+ "b.JP_VERUMIDENTIFIER "
+ "FROM SCUSER.PROBSUMMARYM1 a LEFT JOIN SCUSER.DEVICEM1 b ON (DECODE(a.LOGICAL_NAME,b.LOGICAL_NAME,1,0)=1) "
+ "where a.SEVERITY_CODE IN ('P1/S1','P1/S2','P1/S3') "
+ "and a.CPR_BUSINESS_OPERATIONS = 'f' "
+ "and a.ASSIGNMENT like 'X%' "
+ "and (TRUNC(OPEN_TIME_EST) >=TRUNC(SYSDATE-30) OR TRUNC(CLOSE_TIME_EST) >=TRUNC(SYSDATE-30)) and (a.JP_CONFIDENTIAL IS NULL OR a.JP_CONFIDENTIAL= 'f')";
logger.info(cprSql);
return oracleJdbcTemplate.query(cprSql, new IncidentMapper());
The issue is when i am running these files in local i am able to get the data from oracle to mysql(MariaDB) but when i deploy the same in cloud(Gaia) I am getting a conflict. The Conflict is that the cprSql query in Repository class is connecting to mysql in gaia environment.
Below is the issue when i access the url
Mon Oct 15 12:21:49 UTC 2018
There was an unexpected error (type=Internal Server Error, status=500).
StatementCallback; bad SQL grammar [
SELECT a.NUMBERPRGN as INCIDENT_NUMBER,a.JP_ACCT_APP_APPID as APPLICATION_ID,
a.CATEGORY,to_char(a.OPEN_TIME_EST,'DD-MON-YYYY hh24:mi:ss') OPEN_TIME,
a.SEVERITY_CODE,a.ASSIGNMENT,a.STATUS,
to_char(a.CLOSE_TIME_EST, 'DD-MON-YYYY hh24:mi:ss') CLOSE_TIME,
to_char(a.ELAPSED_TIME, 'DD-MON-YYYY hh24:mi:ss') ELAPSED_TIME,
a.RESOLUTION_CODE, a.TYPE,a.OPEN_GROUP,a.RESOLVED_GROUP,
to_char(a.RESOLVED_TIME, 'DD-MON-YYYY hh24:mi:ss') RESOLVED_TIME,
a.USER_PRIORITY, a.JP_ACCT_LOB,a.JP_ACCT_APP_RTO,a.JP_IMPACT,
a.jp_impact_duration_min, a.JP_OUTAGE_DURATION_MIN,a.JP_TTR_D1,
a.JP_TTR_D2,a.JP_TTR_R1, a.JP_TTR_R2,a.JP_TTR_D2R2,a.LOGICAL_NAME,
b.JP_EXTERNAL_ID, b.JP_EXTERNAL_SYSTEM,b.JP_CONFIG_ITEM,b.JP_MASTER_HOST,
b.JP_SITE,b.JP_APPSERVICE_NAME,b.JP_APPSERVICE_ID,b.JP_VERUMIDENTIFIER
FROM SCUSER.PROBSUMMARYM1 a
LEFT JOIN SCUSER.DEVICEM1 b ON (DECODE(a.LOGICAL_NAME,b.LOGICAL_NAME,
1,0)=1
)
where a.SEVERITY_CODE IN ('P1/S1','P1/S2','P1/S3')
and a.CPR_BUSINESS_OPERATIONS = 'f'
and a.ASSIGNMENT like 'X%'
and (TRUNC(OPEN_TIME_EST) >=TRUNC(SYSDATE-30)
OR TRUNC(CLOSE_TIME_EST) >=TRUNC(SYSDATE-30)
)
and (a.JP_CONFIDENTIAL IS NULL
OR a.JP_CONFIDENTIAL= 'f'
)
]; nested exception is com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: Incorrect parameter count in the call to native function 'DECODE'
Could someone please help ..I am stuck with this from morning!!..
This question already has an answer here:
jdbc to MYSQL error: No suitable driver found for jdbc:mysql://localhost:3306/test?user='root'&password='' [duplicate]
(1 answer)
Closed 7 years ago.
I've put mysql driver in .../ROOT/WEB-INF/lib and Tomcat 8.0/lib but it has no effect. I'm using the following class to connect to DB:
package db;
import java.sql.*;
public class ConnectToDB implements AutoCloseable {
Connection con;
public ConnectToDB(String server, String database, String user,
String password) throws SQLException {
con = DriverManager.getConnection("jdbc:mysql://" + server + "/"
+ database, user, password);
}
#Override
public void close() throws SQLException {
con.close();
}
public Connection getConnection() {
return con;
}
}
The following classes are included in the Eclipse project:
1. src/nr3/model
package nr3.model;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import db.ConnectToDB;
public class MusikkHandler {
private ConnectToDB db;
private Connection con;
private String tableName;
private PreparedStatement pstmtGetRows;
public MusikkHandler(String server, String database, String user,
String password) throws SQLException {
db = new ConnectToDB(server, database, user, password);
con = db.getConnection();
tableName = "album";
}
public void close() throws SQLException {
db.close();
}
public ArrayList<Album> getRows(String genre) throws SQLException {
ArrayList<Album> list = new ArrayList<Album>();
pstmtGetRows = con.prepareStatement("SELECT * FROM " + tableName
+ " WHERE SJANGER = ?");
pstmtGetRows.setString(1, genre);
ResultSet rs = pstmtGetRows.executeQuery();
while (rs.next()) {
list.add(new Album(rs.getString(1), rs.getString(2), rs.getInt(3),
rs.getInt(4), rs.getString(5)));
}
rs.close();
pstmtGetRows.close();
return list;
}
}
-
package nr3.model;
public class Album {
private String tittel;
private String artist;
private int spor;
private int utgitt;
private String sjanger;
public Album (String artist, String tittel, int spor, int utgitt, String sjanger){
setArtist(artist);
setTittel(tittel);
setUtgitt(utgitt);
setSpor(spor);
setSjanger(sjanger);
}
public Album(){
this(null, null, 0, 0, null);
}
public void setTittel(String tittel){
this.tittel = tittel;
}
public void setArtist(String artist){
this.artist = artist;
}
public void setSpor(int spor) {
this.spor = spor;
}
public void setUtgitt(int utgitt) {
this.utgitt = utgitt;
}
public void setSjanger(String sjanger) {
this.sjanger = sjanger;
}
public String getTittel() {
return tittel;
}
public String getArtist() {
return artist;
}
public int getSpor() {
return spor;
}
public int getUtgitt() {
return utgitt;
}
public String getSjanger() {
return sjanger;
}
public String toString(){
return getTittel() + " (" + getArtist() + ")" + " Utgitt: "
+ getUtgitt();
}
}
2. src/nr3/servlets
package nr3.servlets;
import java.io.IOException;
import java.io.PrintWriter;
import java.sql.SQLException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import nr3.model.Album;
import nr3.model.MusikkHandler;
public class MusikkValg extends HttpServlet {
private static final long serialVersionUID = 428937262021570370L;
public void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String genre = request.getParameter("sjanger");
response.setCharacterEncoding("UTF-8");
response.setContentType("text/html");
PrintWriter out = response.getWriter();
MusikkHandler mh;
out.println("<html>");
out.println("<head>");
out.println("<title>Musikk</title>");
out.println("</head>");
out.println("<body>");
out.println("<h1>MUSIKK-ANBEFALINGER</h1>");
out.println("<br/>");
out.println("<br/>");
out.println("<h3>Da bør du kanskje forsøke en av disse:</h3>");
out.println("<br/>");
out.println("<br/>");
try {
mh = new MusikkHandler(" ", " ", " ", " ");
for (Album a : mh.getRows(genre))
out.println("<p>" + a + "</p>");
} catch (SQLException e) {
e.printStackTrace(out);
}
out.println("</body>");
out.println("</html>");
}
}
Update: I've got the following error stacktrace in the browser:
java.sql.SQLException: No suitable driver found for
jdbc:mysql://localhost/pg3100 at
java.sql.DriverManager.getConnection(Unknown Source) at
java.sql.DriverManager.getConnection(Unknown Source) at
db.ConnectToDB.(ConnectToDB.java:10) at
nr3.model.MusikkHandler.(MusikkHandler.java:20) at
nr3.servlets.MusikkValg.doGet(MusikkValg.java:39) at
javax.servlet.http.HttpServlet.service(HttpServlet.java:618) at
javax.servlet.http.HttpServlet.service(HttpServlet.java:725) at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:291)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:52)
at
org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:239)
at
org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
at
org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
at
org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:106)
at
org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
at
org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:142)
at
org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:79)
at
org.apache.catalina.valves.AbstractAccessLogValve.invoke(AbstractAccessLogValve.java:610)
at
org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:88)
at
org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:534)
at
org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1081)
at
org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:658)
at
org.apache.coyote.http11.Http11AprProtocol$Http11ConnectionHandler.process(Http11AprProtocol.java:277)
at
org.apache.tomcat.util.net.AprEndpoint$SocketProcessor.doRun(AprEndpoint.java:2381)
at
org.apache.tomcat.util.net.AprEndpoint$SocketProcessor.run(AprEndpoint.java:2370)
at java.util.concurrent.ThreadPoolExecutor.runWorker(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
at
org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.lang.Thread.run(Unknown Source)
First, you must register your driver at the beginning so DriverManager can user it when getting a connection. It might vary on the implementation.
DriverManager.registerDriver (new com.mysql.jdbc.Driver());
or
Class.forName("com.mysql.jdbc.Driver");
Then you can perform a getConnection() because you'll have a driver registered to be used by DriverManager.
Second: /webapps/ROOT/WEB-INF/lib is a different context and its libraries won't be available for your app unless you are setting the context path of your app as shown in this question. If you wanna add it, try placing your JDBC driver on /webapps/<yourapp>/lib first. Tomcat lib should work as well (but it's not nice when you distribute your app, might conflict with other apps deployed there, using different versions of the driver, etc.)
Third: When asking here, try to reduce your verifiable example to something readable. There's a lot of code which is not relevant to your problem. Reducing makes it easier to read your question and provide help.
I am writing a Java application which reads the data from MySQL and stores it in Cassandra as Sqoop does not support a direct import to Cassandra. I am using Producer-Consumer framework to achieve the same due to high number of records (in millions) in MySQL. But I am getting ReadTimeOut Exception (com.datastax.driver.core.exceptions.DriverException: Timeout during read). I have one Producer class which reads the data from MySQL and puts it into one queue. There is one consumer class which reads the data from that queue and pushes it to Cassndra. There is one manager class which acts as a coordination bridge between these two classes.
Producer class :-
public class MySQLPrintJobProducer implements Runnable {
private BlockingQueue<PrintJobDAO> printerJobQueue = null;
private Connection conn = null;
public MySQLPrintJobProducer(BlockingQueue<PrintJobDAO> printerJobQueue) throws MySQLClientException {
this.printerJobQueue = printerJobQueue;
connect();
}
private void connect() throws MySQLClientException {
try {
Class.forName(MySQLClientConstants.MYSQL_JDBC_DRIVER);
conn = DriverManager.getConnection("jdbc:mysql://mysqlserverhose/mysqldb?user=mysqluser&password=mysqlpasswd");
} catch (ClassNotFoundException e) {
throw new MySQLClientException(ExceptionUtils.getStackTrace(e));
} catch (SQLException e) {
throw new MySQLClientException(ExceptionUtils.getStackTrace(e));
}
}
public void run() {
ResultSet rs = null;
Statement stmt = null;
PreparedStatement pStmt = null;
try {
stmt = conn.createStatement();
// Get total number of print jobs stored.
rs = stmt.executeQuery(MySQLClientConstants.PRINT_JOB_COUNT_QUERY);
int totalPrintJobs = 0;
if(rs != null) {
while(rs.next()) {
totalPrintJobs = rs.getInt(1);
}
}
// Determine the number of iterations.
int rowOffset = 1;
int totalIteration = ((totalPrintJobs / ExportManagerConstants.DATA_TRANSFER_BATCH_SIZE) + 1);
pStmt = conn.prepareStatement(MySQLClientConstants.PRINT_JOB_FETCH_QUERY);
int totalRecordsFetched = 0;
// Iterate over to fetch Print Job Records in bathces and put it into the queue.
for(int i = 1; i <= totalIteration; i++) {
pStmt.setInt(1, rowOffset);
pStmt.setInt(2, ExportManagerConstants.DATA_TRANSFER_BATCH_SIZE);
System.out.println("In iteration : " + i + ", Row Offset : " + rowOffset);
rs = pStmt.executeQuery();
synchronized (this.printerJobQueue) {
if(this.printerJobQueue.remainingCapacity() > 0) {
while(rs.next()) {
totalRecordsFetched = rs.getRow();
printerJobQueue.offer(new PrintJobDAO(rs.getInt(1), rs.getInt(2), rs.getString(3), rs.getDate(4),
rs.getTimestamp(5), rs.getInt(6), rs.getInt(7), rs.getInt(8), rs.getInt(9),
rs.getInt(10), rs.getFloat(11), rs.getFloat(12), rs.getInt(13), rs.getFloat(14), rs.getInt(15),
rs.getDouble(16), rs.getDouble(17), rs.getDouble(18), rs.getDouble(19), rs.getDouble(20),
rs.getFloat(21)));
this.printerJobQueue.notifyAll();
}
System.out.println("In iteration : " + i + ", Records Fetched : " + totalRecordsFetched +
", Queue Size : " + printerJobQueue.size());
rowOffset += ExportManagerConstants.DATA_TRANSFER_BATCH_SIZE;
} else {
System.out.println("Print Job Queue is full, waiting for Consumer thread to clear.");
this.printerJobQueue.wait();
}
}
}
} catch (SQLException e) {
System.err.println(ExceptionUtils.getStackTrace(e));
} catch (InterruptedException e) {
System.err.println(ExceptionUtils.getStackTrace(e));
} finally {
try {
if(null != rs) {
rs.close();
}
if(null != stmt) {
stmt.close();
}
if(null != pStmt) {
pStmt.close();
}
} catch (SQLException e) {
System.err.println(ExceptionUtils.getStackTrace(e));
}
}
ExportManager.setProducerCompleted(true);
}
}
Consumer Class :-
public class CassandraPrintJobConsumer implements Runnable {
private Cluster cluster = null;
private Session session = null;
private BlockingQueue<PrintJobDAO> printerJobQueue = null;
public CassandraPrintJobConsumer(BlockingQueue<PrintJobDAO> printerJobQueue) throws CassandraClientException {
this.printerJobQueue = printerJobQueue;
cluster = Cluster.builder().withPort(9042).addContactPoint("http://cassandrahost").build();
}
public void run() {
int printJobConsumed = 0;
int batchInsertCount = 1;
if(cluster.isClosed()) {
connect();
}
session = cluster.connect();
PreparedStatement ps = session.prepare(CassandraClientConstants.INSERT_PRINT_JOB_DATA);
BatchStatement batch = new BatchStatement();
synchronized (this.printerJobQueue) {
while(true) {
if(!this.printerJobQueue.isEmpty()) {
for(int i = 1; i <= ExportManagerConstants.DATA_TRANSFER_BATCH_SIZE; i++) {
PrintJobDAO printJob = printerJobQueue.poll();
batch.add(ps.bind(printJob.getJobID(), printJob.getUserID(), printJob.getType(), printJob.getGpDate(), printJob.getDateTimes(),
printJob.getAppName(), printJob.getPrintedPages(), printJob.getSavedPages(), printJob.getPrinterID(), printJob.getWorkstationID(),
printJob.getPrintedCost(), printJob.getSavedCost(), printJob.getSourcePrinterID(), printJob.getSourcePrinterPrintedCost(),
printJob.getJcID(), printJob.getCoverageC(), printJob.getCoverageM(), printJob.getCoverageY(), printJob.getCoverageK(),
printJob.getCoverageTotal(), printJob.getPagesAnalyzed()));
printJobConsumed++;
}
session.execute(batch);
System.out.println("After Batch - " + batchInsertCount + ", record insert count : " + printJobConsumed);
batchInsertCount++;
this.printerJobQueue.notifyAll();
} else {
System.out.println("Print Job Queue is empty, nothing to export.");
try {
this.printerJobQueue.wait();
} catch (InterruptedException e) {
System.err.println(ExceptionUtils.getStackTrace(e));
}
}
if(ExportManager.isProducerCompleted() && this.printerJobQueue.isEmpty()) {
break;
}
}
}
}
}
Manager Class :-
public class ExportManager {
private static boolean isInitalized = false;
private static boolean producerCompleted = false;
private static MySQLPrintJobProducer printJobProducer = null;
private static CassandraPrintJobConsumer printJobConsumer = null;
private static BlockingQueue<PrintJobDAO> printJobQueue = null;
public static boolean isProducerCompleted() {
return producerCompleted;
}
public static void setProducerCompleted(boolean producerCompleted) {
ExportManager.producerCompleted = producerCompleted;
}
private static void init() throws MySQLClientException, CassandraClientException {
if(!isInitalized) {
printJobQueue = new LinkedBlockingQueue<PrintJobDAO>(ExportManagerConstants.DATA_TRANSFER_BATCH_SIZE * 2);
printJobProducer = new MySQLPrintJobProducer(printJobQueue);
printJobConsumer = new CassandraPrintJobConsumer(printJobQueue);
isInitalized = true;
}
}
public static void exportPrintJobs() throws ExportException {
try {
init();
} catch (MySQLClientException e) {
throw new ExportException("Print Job Export failed.", e);
} catch (CassandraClientException e) {
throw new ExportException("Print Job Export failed.", e);
}
Thread producerThread = new Thread(printJobProducer);
Thread consumerThread = new Thread(printJobConsumer);
consumerThread.start();
producerThread.start();
}
}
TestNG class :-
public class TestExportManager {
#Test
public void testExportPrintJobs() {
try {
ExportManager.exportPrintJobs();
Thread.currentThread().join();
} catch (ExportException e) {
Assert.fail("ExportManager.exportPrintJobs() failed.", e);
} catch (InterruptedException e) {
Assert.fail("ExportManager.exportPrintJobs() failed.", e);
}
}
}
I have also made some configuration changes by following this link. Still I am getting following exception after inserting 18000 - 20000 records.
Exception in thread "Thread-2" com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /192.168.10.80
(com.datastax.driver.core.exceptions.DriverException: Timeout during read))
at com.datastax.driver.core.exceptions.NoHostAvailableException.copy(NoHostAvailableException.java:64)
at com.datastax.driver.core.DefaultResultSetFuture.extractCauseFromExecutionException(DefaultResultSetFuture.java:256)
at com.datastax.driver.core.DefaultResultSetFuture.getUninterruptibly(DefaultResultSetFuture.java:172)
at com.datastax.driver.core.SessionManager.execute(SessionManager.java:91)
at com.incendiary.ga.client.cassandra.CassandraPrintJobConsumer.run(CassandraPrintJobConsumer.java:108)
at java.lang.Thread.run(Unknown Source)
Caused by: com.datastax.driver.core.exceptions.NoHostAvailableException: All host(s) tried for query failed (tried: /192.168.10.80 (com.datastax.drive
r.core.exceptions.DriverException: Timeout during read))
at com.datastax.driver.core.RequestHandler.sendRequest(RequestHandler.java:100)
at com.datastax.driver.core.RequestHandler$1.run(RequestHandler.java:171)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(Unknown Source)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(Unknown Source)
... 1 more
I am not able to figure out the actual reason for the issue. I could not find any exception in Cassandra system log. I am using Apache Cassandra 2.0.7 and cassandra-driver-core 2.0.1.
You can increase read time out in you driver side . By using withSocket method in this you have SocketOption class using that you can read time out .By default is read time out is 10 millisecond .
I had the period webservice originally tomcat server.
Now I want to segment webservice transplant to GAE.
I transplant was successful.
Implementation, but this error message is displayed.
Will this cause the possibility of error?
MY CODE:
public class LOAD extends HttpServlet{
static String JBx;
static String JBy;
static String JBSx;
static String JBSy;
static String gps;
String a;
int i;
private Connection con = null; //Database objects
private Statement stat = null;
private ResultSet rs = null;
private PreparedStatement pst = null;
private String dropdbSQL = "DROP TABLE User ";
private String createdbSQL = "CREATE TABLE User (" +
" id INTEGER " +
" , name VARCHAR(20) " +
" , passwd VARCHAR(20))";
private String insertdbSQL = "insert into User(id,gps,recopy) " +
"select ifNULL(max(id),0)+1,?,? FROM User";
private String selectSQL = "select * from User ";
public static void main(String[] args)
{
}
public void SelectTable()
{
try
{
stat = con.createStatement();
rs = stat.executeQuery(selectSQL);
while(rs.next())
{
JBSx =rs.getString("gps");
JBSx="<div id=dd>"+JBSx+"</div>";
if(JBx!=null){
JBx=JBx+JBSx;
}else
{
JBx=JBSx;
}
JBSy =rs.getString("recopy");
JBSy="<div id=ff>"+JBSy+"</div>";
if(JBy!=null){
JBy=JBy+JBSy;
}
else
{
JBy=JBSy;
}
}
}
catch(SQLException e)
{
System.out.println("DropDB Exception :" + e.toString());
}
// finally
// {
// Close();
// }
}
private void Close()
{
try
{
if(rs!=null)
{
rs.close();
rs = null;
}
if(stat!=null)
{
stat.close();
stat = null;
}
if(pst!=null)
{
pst.close();
pst = null;
}
}
catch(SQLException e)
{
System.out.println("Close Exception :" + e.toString());
}
}
public LOAD()
{
try {
Class.forName("com.mysql.jdbc.Driver");
con = DriverManager.getConnection(
"jdbc:mysql://hl2dm.synology.me:3306/teama",
"abc","");
}
catch(ClassNotFoundException e)
{
System.out.println("DriverClassNotFound :"+e.toString());
}
catch(SQLException x) {
System.out.println("Exception :"+x.toString());
}
}
public void doGet(HttpServletRequest rq, HttpServletResponse rp)
throws ServletException, IOException{
JBx=null;
JBy=null;
JBSx=null;
JBSy=null;
rq.setCharacterEncoding("UTF-8");
// String gps = rq.getParameter("gps");
LOAD test = new LOAD();
test.SelectTable();
HttpSession session = rq.getSession();
//session.invalidate();
// session.removeAttribute("reportx");
session.setMaxInactiveInterval(3600);
session.setAttribute("reportx",JBx);
session.setAttribute("reporty",JBy);
rp.sendRedirect(rp.encodeRedirectURL("LOADPOST"));
}
public void doPost(HttpServletRequest rq, HttpServletResponse rp)
throws ServletException, IOException{
doGet(rq, rp);
}
}
Error message:
HTTP ERROR 500
Problem accessing /LOAD. Reason:
INTERNAL_SERVER_ERROR
Caused by:
java.lang.NullPointerException
at a.b.c.LOAD.SelectTable(LOAD.java:52)
at a.b.c.LOAD.doGet(LOAD.java:149)
at a.b.c.LOAD.doPost(LOAD.java:160)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:637)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
at com.google.appengine.api.socket.dev.DevSocketFilter.doFilter(DevSocketFilter.java:74)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.appengine.tools.development.ResponseRewriterFilter.doFilter(ResponseRewriterFilter.java:123)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.appengine.tools.development.HeaderVerificationFilter.doFilter(HeaderVerificationFilter.java:34)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.appengine.api.blobstore.dev.ServeBlobFilter.doFilter(ServeBlobFilter.java:61)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.apphosting.utils.servlet.TransactionCleanupFilter.doFilter(TransactionCleanupFilter.java:43)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.appengine.tools.development.StaticFileFilter.doFilter(StaticFileFilter.java:125)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.appengine.tools.development.BackendServersFilter.doFilter(BackendServersFilter.java:97)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
at com.google.appengine.tools.development.DevAppEngineWebAppContext.handle(DevAppEngineWebAppContext.java:94)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at com.google.appengine.tools.development.JettyContainerService$ApiProxyHandler.handle(JettyContainerService.java:409)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.content(HttpConnection.java:938)
at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:755)
at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:218)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409)
at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Powered by Jetty://
To answer your question, yes it will cause definite failure.
There is no MySql instance to connect to on AppEngine. You need instead to use Google Cloud SQL.
It goes something like this:
DriverManager.registerDriver(new AppEngineDriver());
con = DriverManager.getConnection("jdbc:google:rdbms://instance_name/guestbook");
See: this AppEngine guide for the complete set of steps.
Note: To use the development server with a local MySQL instance, see https://developers.google.com/appengine/docs/java/cloud-sql/developers-guide#using_the_java_development_server