I have a lot of tables and few tables have around 20 million records.
Now I do some calculations on these tables before showing the result to the UI.
For that I have created stored procedure.
In the stored procedure I am using temprorary tables to store select query records, do join, do processing and then return result.
I have written query like this
Insert INTO A
SELECT * from B JOIN c ....
Now my select query here does not take time (I have optimize it using explain extended) but my Insert into takes a lot of time since the output of select is in million. My table A is temprorary.
And Also I have do some processing on this A table after that.
My query is is there a way where I can skip inserting this insert.
I can take the result in a seperate table variable(if any) and then do processing instead of inserting all that in temprorary and then again doing some processing on it.
Adding my stored procedure here
DROP PROCEDURE IF EXISTS 6_4_1n2_PortUtil_temp;
DELIMITER |
CREATE PROCEDURE 6_4_1n2_PortUtil_temp(utilType VARCHAR(100),service varchar(5000),p_networkType VARCHAR(20),inputCity varchar(5000),inputNodeName varchar(5000), startTime timestamp,endTime timestamp)
BEGIN
DROP TEMPORARY TABLE IF EXISTS TEMP_SERVICE_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL_1;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEName_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEANDIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_ROUTERTRAFFIC_VLANPRT_SCALE1_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_TRAFFIC_TBL;
DROP TEMPORARY TABLE IF EXISTS TRAFFIC_TBL;
DROP TEMPORARY TABLE IF EXISTS FINAL_FETCH;
SELECT now();
CREATE TEMPORARY TABLE TEMP_SERVICE_TBL(nodeName varchar(256),NodeNumber int) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEName_TBL(nodeName varchar(256),NodeNumber int) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEIF_TBL(NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100)) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEIF_TBL_1(NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100)) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_NODEANDIF_TBL(NodeName Varchar(256),NodeNumber int,IfIndex INTEGER,IfSpeed FLOAT,IfDescr VARCHAR(100),IfAlias VARCHAR(100),PortID BIGINT(20) DEFAULT 0) ENGINE INNODB;
CREATE TEMPORARY TABLE TRAFFIC_TBL(
PortID BIGINT(20),
NodeName VARCHAR(100),
IfDescr VARCHAR(100),
IfSpeed VARCHAR(100),
InErrPkts BIGINT(20),
RcvOctets BIGINT(20),
TxOctets BIGINT(20),
Time_1 TIMESTAMP) ENGINE INNODB;
CREATE TEMPORARY TABLE TEMP_TRAFFIC_TBL(
PortID BIGINT(20),
maxTrafficValueIn BIGINT(20) DEFAULT 0,
maxOutTrafficValueOut BIGINT(20) DEFAULT 0,
avgTrafficValueIn BIGINT(20) DEFAULT 0,
avgTrafficValueOut BIGINT(20) DEFAULT 0,
CRCError BIGINT(20) DEFAULT 0,
UpTime INTEGER DEFAULT 0,
Reliablity INTEGER DEFAULT 0,
AvgUtilIn float DEFAULT 0,
AvgUtilOut float DEFAULT 0,
PeakUtilIn float DEFAULT 0,
PeakUtilOut float DEFAULT 0,
ThresholdExceed INTEGER DEFAULT 0,
inPeakTime timestamp DEFAULT '0000-00-00 00:00:00',
outPeakTime timestamp DEFAULT '0000-00-00 00:00:00') ENGINE INNODB;
SET #where = '';
IF service='ALL'
THEN
SET #where = '';
ELSE
set #a=1;
set #like="";
select REPLACE(SUBSTRING(SUBSTRING_INDEX(service, ',', #a),LENGTH(SUBSTRING_INDEX(service, ',', #a -1)) + 1),',','') into #service;
while(#service != "")
DO
IF(#like = "")
THEN
SET #like = CONCAT("NodeName like '%",SUBSTRING(#service,2,3),"%'");
ELSE
SET #like = CONCAT(#like," or NodeName like '%",SUBSTRING(#service,2,3),"%'");
END IF;
set #a=#a+1;
select REPLACE(SUBSTRING(SUBSTRING_INDEX(service, ',', #a),LENGTH(SUBSTRING_INDEX(service, ',', #a -1)) + 1),',','') into #service;
END WHILE;
SET #where = CONCAT(" where",#like);
END IF;
Set #where2 = '';
IF inputCity='ALL'
THEN
set #where2 = '';
ELSE
set #where2 = CONCAT(' and substring(NodeName,1,3) in (',inputCity,')');
END IF;
SET #where3 = '';
IF inputNodeName='ALL'
THEN
set #where3 = '';
ELSE
set #where3 = CONCAT(' and NodeName in (',inputNodeName,')');
END IF;
SET #query1 := CONCAT("INSERT INTO TEMP_NODEName_TBL SELECT distinct NodeName,NodeNumber from NODE_TBL",#where, #where2, #where3);
SELECT #query1;
PREPARE statement1 from #query1;
EXECUTE statement1;
DEALLOCATE Prepare statement1;
CREATE INDEX n1 ON TEMP_NODEName_TBL(NodeNumber);
CREATE INDEX i1 ON TEMP_NODEIF_TBL(NodeNumber,IfIndex);
CREATE INDEX portIDIndex1 on TEMP_NODEANDIF_TBL(PortID);
SET #where4 = '';
IF (utilType='ALL')
THEN
SET #where = '';
ELSE
If (utilType = "'AESI-IN'")
THEN
SET #where4 = " where IfAlias like '%AESI-IN%'";
ELSE
SET #where4 = " where IfAlias NOT like '%AESI-IN%'";
END IF;
END IF;
CREATE INDEX i2 ON TEMP_NODEIF_TBL_1(NodeNumber,IfIndex);
CREATE INDEX i3 ON TEMP_NODEANDIF_TBL(NodeNumber,IfIndex);
SET #where5 = '';
IF(p_networkType != 'ALL')
THEN
set #r1= SUBSTRING(p_networkType,2,3);
set #r2= SUBSTRING(p_networkType,8,3);
set #r3= SUBSTRING(p_networkType,14,3);
SET #where5 = CONCAT(" and IfAlias like '%",#r1,"%'");
if(#r2 != "")
THEN
SET #where5 = CONCAT(" and IfAlias like '%",#r2,"%'");
IF(#r3 != "")
THEN
SET #where5 = CONCAT(" and IfAlias like '%",#r3,"%'");
END IF;
END IF;
END IF;
SET #query2 := CONCAT("INSERT INTO TEMP_NODEANDIF_TBL(NodeName,NodeNumber,IfIndex,IfSpeed,IfDescr,IfAlias) SELECT distinct b.NodeName, a.NodeNumber, a.IfIndex,a.IfSpeed,a.IfDescr,a.IfAlias from NODEIF_TBL a JOIN TEMP_NODEName_TBL b ON a.NodeNumber = b.NodeNumber ", #where4, #where5);
SELECT #query2;
PREPARE statement1 from #query2;
EXECUTE statement1;
DEALLOCATE Prepare statement1;
SELECT "DROP TEMPORARY TABLES";
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEIF_TBL_1;
DROP TEMPORARY TABLE IF EXISTS TEMP_NODEName_TBL;
update TEMP_NODEANDIF_TBL a,VLANPRT_TBL b set a.PortID = PrtID where a.NodeNumber = b.NodeID and a.IfIndex = b.IfIndex;
SELECT "Update Temporary tables";
delete from TEMP_NODEANDIF_TBL where PortID = 0;
SELECT now();
INSERT INTO TRAFFIC_TBL
SELECT a.PortID,NodeName,IfDescr,IfSpeed,InErrPkts,RcvOctets,TxOctets,Time_1 FROM ROUTERTRAFFIC_VLANPRT_SCALE1_TBL a JOIN TEMP_NODEANDIF_TBL b ON a.PortID = b.PortID where Time_1>startTime and Time_1<endTime;
SELECT now();
DROP TEMPORARY TABLE IF EXISTS EXCEED_COUNT;
CREATE TEMPORARY TABLE EXCEED_COUNT (PortID BIGINT(20), Exceed INTEGER);
INSERT INTO EXCEED_COUNT
select PortID,count(RcvOctets) from TRAFFIC_TBL where (RcvOctets/(IfSpeed*10)>70 or TxOctets/(IfSpeed*10)>70) group by PortID;
INSERT INTO TEMP_TRAFFIC_TBL (PortID, maxTrafficValueIn,maxOutTrafficValueOut,avgTrafficValueIn,avgTrafficValueOut,CRCError,AvgUtilIn,AvgUtilOut,PeakUtilIn,PeakUtilOut)
SELECT PortID,max(RcvOctets),max(TxOctets),avg(RcvOctets),avg(TxOctets), sum(InErrPkts),
IF((IfSpeed=0),"0",(avg(RcvOctets)/(IfSpeed*10))),
IF((IfSpeed=0),"0",(avg(TxOctets)/(IfSpeed*10))),
IF((IfSpeed=0),"0",(max(RcvOctets)/(IfSpeed*10))),
IF((IfSpeed=0),"0",(max(TxOctets)/(IfSpeed*10)))
from TRAFFIC_TBL group by PortID;
CREATE INDEX portIDIndex2 on TEMP_TRAFFIC_TBL(PortID);
CREATE INDEX portIDIndex3 on EXCEED_COUNT(PortID);
UPDATE TEMP_TRAFFIC_TBL A JOIN TRAFFIC_TBL B ON A.PortId=B.PortId SET inPeakTime=B.Time_1 where B.RcvOctets=A.maxTrafficValueIn;
UPDATE TEMP_TRAFFIC_TBL A JOIN TRAFFIC_TBL B ON A.PortId=B.PortId SET outPeakTime=B.Time_1 where B.TxOctets=A.maxOutTrafficValueOut;
UPDATE TEMP_TRAFFIC_TBL A JOIN EXCEED_COUNT B ON (A.PortID = B.PortID)
set ThresholdExceed = Exceed;
SELECT substring(NodeName,5,3) as ServiceType,
CASE
WHEN IfAlias like '%SWH%' THEN "Trunk"
WHEN IfAlias like '%AES%' THEN "Backbone"
WHEN IfAlias like '%RTR%' THEN "Back-to-Back"
ELSE "-"
END ,
NodeName,IfDescr,ROUND(maxTrafficValueIn/1000,2),ROUND(maxOutTrafficValueOut/1000,2),ROUND(avgTrafficValueIn/1000,2),ROUND(avgTrafficValueOut/1000,2),ROUND(CRCError/1000,2),0,0,ROUND(AvgUtilIn,2),ROUND(AvgUtilOut,2),ROUND(PeakUtilIn,2),ROUND(PeakUtilOut,2),ThresholdExceed,inPeakTime,outPeakTime from TEMP_TRAFFIC_TBL a ,TEMP_NODEANDIF_TBL b where a.PortID = b.PortID ;
SELECT now();
END |
DELIMITER ;
My ROUTERTRAFFIC_VLANPRT_SCALE1_TBL contains around 20 million records and NodeIF_TBL around 1 lakhs records. My VLANPRT_TBL also contains around 1 lakh records.
Also I have BTree indexing set on time_1 so that the join on the big table do not take time
Temporary tables:
Temporary tables start out in the memory and if the surpass a certain limitation they will be written to disk. If the select yields millions and you're writing those millions to disk then that's your problem.
Needing a temporary table with millions of records is usually a sign of doing something wrong. Perhaps you should further filter the data before inserting it into the temporary table for processing.
Disk I/O thrashing:
We have a few INSERT cases:
INSERT INTO ... VALUES
INSERT INGORE INTO ... SELECT
INSERT INTO ... SELECT
For 1. each insert has its own transaction and writes to disk individually meaning lots of tiny I/O operations.
For 2. the same applies except the data is also being read from the disk (slightly more I/O operations).
For 3. the whole operation is one big transaction meaning one big disk I/O at the very end (the commit) of the transaction. More often it happens that the information contained in the big transaction cannot be held entirely in memory so it is written to disk temporarily before the final commit and then copied from there into the designated area where it should reside.
In all 3 cases disk I/O thrashing occurs, to prevent this you would group small operations into average sized operations, and break huge operations into average sized operations.
For 1. you need to wrap every N inserts in a transaction.
For 2. and 3. you need to LIMIT the SELECT clause to N and repeat the INSERT INTO ... SELECT with an added OFFSET of N until you finish all records.
Either should be fairly easy to do if you're using a scripting language to run your queries.
Its not needed to store in a temporary table the output from a query in order to use it in another query. You can make this:
Select t1.* from (select * from A where condition1)as t1 where condition2
Here is an example of a query i had to do.
select avg(v1) as v1avg ,start_date_time as time, timekey as time_key from (select ATable.*, ROUND(UNIX_TIMESTAMP(start_date_time)/(60*60)) as timekey from ATable where start_date_time between '2014-01-01 00:00:00' and '2014-01-10 00:00:00')as t1 group by timekey;
I have one complex stored procedure that returns some rows with some calculated values
SELECT CalculatedField1 ,
CalculatedField2 ,
...
FROM ...
WHERE CONDITION
this sproc (lets call it procA) returns variable number of rows, depending on the WHERE condition. This works fine. What I need to do now is to write a stored procedure that will get the summary of these returned rows.
-- procB
SELECT SUM(CalculatedField1),
SELECT SUM(CalculatedField2),
...
FROM (EXEC procA params)
Is this possible?
Edit: creating a temp table did the job, however I have problems with passing output pareters.
CREATE PROCEDURE [dbo].[sprocB] (#prm INT = NULL OUTPUT)
AS
BEGIN
SET NOCOUNT ON;
SET #prm = 1
SELECT Id FROM dbo.AnyTable
END
CREATE PROCEDURE [dbo].[sprocA] (#prm INT = NULL OUTPUT)
AS
BEGIN
SET NOCOUNT ON;
CREATE TABLE #temp (Id INT)
INSERT INTO #temp
EXEC sprocB #prm
SELECT Id FROM #temp
END
To execute:
USE [MyDatabase]
GO
DECLARE #return_value int,
#prm int
EXEC #return_value = [dbo].[sprocA]
#prm = #prm OUTPUT
SELECT #prm as N'#prm'
SELECT 'Return Value' = #return_value
GO
The results from temp table are ok, resultset is retrieved correctly, however #pem value is still NULL.
It is possible but you need an auxiliary table:
CREATE PROCEDURE procB
AS
declare #table table (CalculatedField1 int, CalculatedField12 int)
insert into #table
EXEC sp_a
select SUM(CalculatedField1), SUM(CalculatedField2)
from #table
GO
This might do what you want:
create table #scratch (CalculatedField1 int, CalculatedField2 int,...)
insert into #scratch (exec procA params)
select sum(CalculatedField1), sum(CalculatedField2),... from #scratch
drop table #scratch
So suppose I have this table with 4 columns:
id content parent timestamp
whereby the parent column refers to an id of another entry in the table
I want to accomplish the following:
Select the first 50 rows from the table Ordered by the following:
for each row,
if(parent = 0){
add row to resultset, ordered by timestamp
}
else if (parent != 0){
if parent is in the list of rows already fetched so far by the query,
add row to resultset, ordered by the timestamp
otherwise, wait until the parent gets fetched by the query
(assuming it gets fetched at all since there we're only getting the first 50 rows)
}
this ordering logic is somewhat complicated, and I'm wondering if it's even possible to accomplish this using MYSQL ORDER BY statement in a single query WITHOUT having to resort to subqueries? Perhaps we could set and use variables? But how would the ORDER BY statement will be implemented?
Here's a solution using variables which I incorporated into a procedure with a loop using an outer join and a temporary table to hold the unused inputdata but no subqueries.
Delete any old version of the procedure and set the delimiter
DROP PROCEDURE IF EXISTS order_proc;
DELIMITER ;;
Start writing the procedure
CREATE PROCEDURE order_proc()
BEGIN
DECLARE n INT DEFAULT 50;
DECLARE m INT DEFAULT 0;
DECLARE i INT DEFAULT 0;
DECLARE custom_limit INT DEFAULT 0;
DROP TABLE IF EXISTS pre_resultset;
CREATE TABLE pre_resultset LIKE input_data;
ALTER TABLE pre_resultset MODIFY id INT;
ALTER TABLE pre_resultset DROP PRIMARY KEY;
ALTER TABLE pre_resultset ADD COLUMN iid INT PRIMARY KEY NOT NULL AUTO_INCREMENT FIRST;
SELECT n INTO custom_limit;
Set SQL_SELECT_LIMIT = custom_limit;
INSERT INTO pre_resultset SELECT NULL,id, content, parent, timestamp FROM input_data WHERE parent = 0 ORDER BY timestamp;
Set SQL_SELECT_LIMIT = default;
DROP TABLE IF EXISTS unused_input_data;
CREATE TABLE unused_input_data LIKE input_data;
ALTER TABLE unused_input_data ADD null_col VARCHAR(1);
SELECT COUNT(*) FROM pre_resultset INTO m;
WHILE m<n DO
SELECT COUNT(*) FROM pre_resultset INTO m;
TRUNCATE unused_input_data;
INSERT INTO unused_input_data SELECT input_data.id, input_data.content, input_data.parent, input_data.timestamp, pre_resultset.id AS null_col FROM input_data LEFT OUTER JOIN pre_resultset on input_data.id = pre_resultset.id WHERE pre_resultset.id IS NULL ;
SELECT n-m INTO custom_limit;
Set SQL_SELECT_LIMIT = custom_limit;
INSERT INTO pre_resultset SELECT NULL, unused_input_data.id, unused_input_data.content, unused_input_data.parent, unused_input_data.timestamp FROM unused_input_data JOIN pre_resultset WHERE unused_input_data.parent = pre_resultset.id ORDER BY unused_input_data.timestamp;
Set SQL_SELECT_LIMIT = default;
SELECT COUNT(*) FROM pre_resultset INTO i;
SELECT (IF( i = m, n, i)) INTO m;
END WHILE;
SELECT id, content, parent, timestamp FROM pre_resultset AS resultset;
DROP TABLE IF EXISTS pre_resultset;
DROP TABLE IF EXISTS unused_input_data;
End;
;;
Change the delimeter back
DELIMITER ;
Run the procedure
CALL order_proc();
I have a table which contains header information for transactions. The transactions belong to different projects.
In the header I have columns:
rhguid - uniqueidentifier
rhserial - int
rh_projectID - int
First I insert the row (there's more columns)
Then I calculate the serial number for that project:
update responseheader
set rhSerial = 1 + (select isnull(max(rhSerial), 0)
from responseheader
where (rhstatus = 0) AND (rh_projectID = 1234))
where
(rhGUID = <preassignedGUID>);
However when there are many transactions happening at the same time for a project I am finding duplicate rhserial values.
I'm doing this in classic ASP with SQL Server 2008.
Is there a better way?
From your example, it doesn't look like you're using a transaction. My guess is that the SELECT portion of the statement is running as READ UNCOMMITTED, otherwise you would not see duplicates. There are ways to start transactions with ADO, but I prefer using stored procedures instead.
Try implementing something like this:
CREATE PROC dbo.ResponseHeader_Insert
<more data to insert>,
#ProjectID INT,
#Status SMALLINT
as
insert responseheader (column names here)
select <param values here>, isnull(max(rhSerial), 0) + 1
from responseheader
where (rhstatus = #Status) AND (rh_projectID = #ProjectID))
If this doesn't work for ya, try creating sequence tables (one for each sequence).
create table <tablename> (
SeqID int identity(1,1) primary key,
SeqVal varchar(1)
)
Create a procedure to get the next identity:
create procedure GetNewSeqVal_<tablename>
as
begin
declare #NewSeqValue int
set NOCOUNT ON
insert into <tablename> (SeqVal) values ('a')
set #NewSeqValue = scope_identity()
delete from <tablename> WITH (READPAST)
return #NewSeqValue
end
If there are too many sequence tables that need to be created or you want to create sequences on the fly, try this approach:
Create table AllSequences (
SeqName nvarchar(255) primary key, -- name of the sequence
Seed int not null default(1), -- seed value
Incr int not null default(1), -- incremental
Currval int
)
Go
create procedure usp_CreateNewSeq
#SeqName nvarchar(255),
#seed int = 0,
#incr int = 1
as
begin
declare #currval int
if exists (
select 1 from AllSequences
where SeqName = #SeqName )
begin
print 'Sequence already exists.'
return 1
end
if #seed is null set #seed = 1
if #incr is null set #incr = 1
set #currval = #seed
insert into AllSequences (SeqName, Seed, Incr, CurrVal)
values (#SeqName, #Seed, #Incr, #CurrVal)
end
go
create procedure usp_GetNewSeqVal
#SeqName nvarchar(255)
as
begin
declare #NewSeqVal int
set NOCOUNT ON
update AllSequences
set #NewSeqVal = CurrVal = CurrVal+Incr
where SeqName = #SeqName
if ##rowcount = 0 begin
print 'Sequence does not exist'
return
end
return #NewSeqVal
end
go