BEGIN
declare currID INT(30);
declare maxID INT(30);
declare first_period datetime;
declare s_date datetime;
declare tot_sent INT(30);
declare weighted_score FLOAT(30);
declare aid INT(10);
set first_period = date_add(stats_start_from, INTERVAL 49 DAY);
set currID = (select min(rowid) from weekly_stats);
set maxID = (select max(rowid) from weekly_stats);
WHILE (currID <= 40000) DO
select accountid, week_start_date INTO aid, s_date from weekly_stats where rowid = currID;
if (s_date >= first_period) then
set tot_sent = (select sum(weekly_total_sent) from weekly_stats where (accountid = aid AND week_start_date <= s_date AND week_start_date >= (date_add(s_date, INTERVAL (-49) DAY))));
set weighted_score = (select sum(weekly_total_sent*weekly_raw_score) from weekly_stats where (accountid = aid AND week_start_date <= s_date AND week_start_date >= (date_add(s_date, INTERVAL (-49) DAY))));
set weighted_score = weighted_score/tot_sent;
update weekly_stats set weighted_scores = weighted_score where rowid = currID;
end if;
set currID = currID + 1;
end while;
END
Basically, I have a column in the table called weighted_scores that calculates as follows: weighted score = sum of (raw score*sent mail) for the last 8 weeks, divided by the total mail sent in the last 8 weeks. So, it is a 8 week score average weighted by the volume of mail sent each week.
My procedure runs really slowly and times out (after ten minutes) for even a small number of rows. I was wondering what the issue was, maybe inefficient querying, calculations, etc? Any ideas would be much appreciated. Thanks!
create table weekly_stats
(
accountid INT(10),
week_start_date datetime,
weekly_total_sent INT(30),
weekly_hbounce INT(30),
weekly_sbounce INT(30),
weekly_spam INT(30),
weekly_optouts INT(30),
weekly_opens INT(30),
weekly_CT INT(30),
rowid INT(30),
weekly_raw_score FLOAT(30),
weighted_scoresFLOAT(30)
-- indexes not shown
);
Related
I need to create a temporary table that is populated based on two parameters:
declare #Start date = '01/01/2015'
declare #End date = '12/31/2015'
The temporary table should have a column that will capture YYYYMM for all the years and month that are between #Start and #End parameter.
Here's what I have so far. I want to stop it at 201412 and then start again at 201501. Instead, this loop keeps going in increment of plus 1 (I do not want to see 201413..so on):
declare #Start date = '01/01/2014'
declare #End date = '12/31/2015'
declare #monthstart as int
declare #monthend as int
declare #increment as int
set #monthstart = (SELECT LEFT(CONVERT(varchar, #Start,112),6))
set #monthend = (SELECT LEFT(CONVERT(varchar, #End,112),6))
create table #datetemp (RelevantYYYYMM int)
insert into #datetemp values (#monthstart)
set #increment = #monthstart
While #increment < #monthend
BEGIN
set #increment = (select Max(RelevantYYYYMM) + 1 from #datetemp)
insert into #datetemp values (#increment)
set #increment = (select Max(RelevantYYYYMM) from #datetemp)
IF (select Max(RelevantYYYYMM) from #datetemp) > #monthend
Break
else
continue
END
select * from #datetemp
You can use tally table and avoid loop:
CREATE TABLE #datetemp (RelevantYYYYMM INT);
DECLARE #Start DATE = '01/01/2015', #End DATE = '12/31/2015';
WITH tally_table AS
(
SELECT TOP 1000 rn = ROW_NUMBER() OVER(ORDER BY name) - 1
FROM master..spt_values
)
INSERT INTO #datetemp(RelevantYYYYMM)
SELECT LEFT(CONVERT(varchar, DATEADD(month, rn, #Start),112),6)
FROM tally_table
WHERE YEAR(DATEADD(month, rn, #Start)) <= YEAR(#End)
AND MONTH(DATEADD(month, rn, #Start)) <= MONTH(#End)
SELECT *
FROM #datetemp;
LiveDemo
I've been working on optimizing a query which compresses tick price data into O-H-L-C intervals. I'm trying to accomplish this with a single query, instead of having to use multiple queries with row partitioning to determine the Open and Close.
With the help of a great response to a question I posted yesterday, I've come up with this so far:
DECLARE #Interval INT = 5
DECLARE #InstrumentId INT = 36
DECLARE #Start_Date DATETIME = '2015-01-01'
DECLARE #End_Date DATETIME = '2015-03-30'
DECLARE #OffsetTime DATETIME = 0
SELECT INSTRUMENT_ID,
DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime) INTERVAL_TIME,
SUBSTRING(MIN(CONVERT(VARCHAR(24),[TIME_STAMP],21) + '_' +
CAST(RATE_BID AS VARCHAR(10))),25,8) [OPEN],
MAX(RATE_BID) HIGH,
MIN(RATE_BID) LOW,
SUBSTRING(MAX(CONVERT(VARCHAR(24),[TIME_STAMP],21) + '_' +
CAST(RATE_BID AS VARCHAR(10))),25,8) [CLOSE]
FROM dbo.TICKS
WHERE INSTRUMENT_ID = #InstrumentId AND TIME_STAMP BETWEEN #Start_Date AND #End_Date
GROUP BY DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime),
INSTRUMENT_ID
ORDER BY INTERVAL_TIME
Is there a more efficient way to concatenate the date/time to the price, then extract only the price once the MIN and MAX is evaluated? I was thinking about a binary representation of the date/time , adding it to the price , then some bit operation to extract the price. I'm not quite sure where to begin.
To test the performance of a query, it would be useful to have DDL and sample data. I am assuming the following table structure:
CREATE TABLE TICKS (
INSTRUMENT_ID INT,
TIME_STAMP DATETIME,
PRIMARY KEY (INSTRUMENT_ID, TIME_STAMP),
RATE_BID INT NOT NULL
)
To generate some sample data, I used the following code:
CREATE FUNCTION dbo.Numbers(#N int)
RETURNS TABLE AS RETURN
WITH
L0 AS(SELECT 1 AS C UNION ALL SELECT 1 AS O), -- 2 rows
L1 AS(SELECT 1 AS C FROM L0 AS A CROSS JOIN L0 AS B), -- 4 rows
L2 AS(SELECT 1 AS C FROM L1 AS A CROSS JOIN L1 AS B), -- 16 rows
L3 AS(SELECT 1 AS C FROM L2 AS A CROSS JOIN L2 AS B), -- 256 rows
L4 AS(SELECT 1 AS C FROM L3 AS A CROSS JOIN L3 AS B), -- 65,536 rows
L5 AS(SELECT 1 AS C FROM L4 AS A CROSS JOIN L4 AS B), -- 4,294,967,296 rows
Nums AS(SELECT ROW_NUMBER() OVER(ORDER BY (SELECT NULL)) AS Number FROM L5)
SELECT Number FROM Nums WHERE Number<=#N
GO
--DELETE dbo.TICKS
SET NOCOUNT ON
DECLARE #INSTRUMENT_ID INT
SET #INSTRUMENT_ID=1
WHILE #INSTRUMENT_ID<50 BEGIN
DECLARE RandomData CURSOR LOCAL READ_ONLY FOR
SELECT TIME_STAMP, CONVERT(INT,RAND(CHECKSUM(NEWID()))*10) AS Delta,
CONVERT(BIT,CONVERT(INT,RAND(CHECKSUM(NEWID()))*1.1)) AS ChangeDirection
FROM (
--SELECT DATEADD(MINUTE,Number,'20150101') AS TIME_STAMP FROM dbo.Numbers(150000)
SELECT DATEADD(SECOND,Number*10,'20150101') AS TIME_STAMP FROM dbo.Numbers(900000)
) x
WHERE DATEPART(HOUR,TIME_STAMP) BETWEEN 8 AND 15
AND (DATEPART(WEEKDAY,TIME_STAMP)+##DATEFIRST)%7>1
AND TIME_STAMP>'20150103'
OPEN RandomData
DECLARE #TIME_STAMP DATETIME, #Delta INT, #ChangeDirection BIT
DECLARE #RATE_BID INT, #Direction SMALLINT
SET #RATE_BID=CONVERT(INT,RAND(CHECKSUM(NEWID()))*10000)+100
SET #Direction=1
WHILE 1=1 BEGIN
FETCH NEXT FROM RandomData INTO #TIME_STAMP, #Delta, #ChangeDirection
IF ##FETCH_STATUS<>0 BREAK
SET #Direction=CASE WHEN #ChangeDirection=1 THEN -#Direction ELSE #Direction END
IF #RATE_BID<100 AND #Direction<0 SET #Direction=1
SET #RATE_BID=#RATE_BID+#Delta*#Direction
INSERT INTO dbo.TICKS VALUES (#INSTRUMENT_ID, #TIME_STAMP, #RATE_BID)
END
CLOSE RandomData
DEALLOCATE RandomData
SET #INSTRUMENT_ID=#INSTRUMENT_ID+1
END
SET NOCOUNT OFF
Then I tested your original query against a variation using binary data types instead of strings:
DECLARE #Interval INT = 5
DECLARE #InstrumentId INT = 36
DECLARE #Start_Date DATETIME = '2015-01-01'
DECLARE #End_Date DATETIME = '2015-03-30'
DECLARE #OffsetTime DATETIME = 0
DECLARE #StartTime DATETIME
SET #StartTime=GETDATE()
SELECT INSTRUMENT_ID,
DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime) INTERVAL_TIME,
SUBSTRING(MIN(CONVERT(VARCHAR(24),[TIME_STAMP],21) + '_' +
CAST(RATE_BID AS VARCHAR(10))),25,8) [OPEN],
MAX(RATE_BID) HIGH,
MIN(RATE_BID) LOW,
SUBSTRING(MAX(CONVERT(VARCHAR(24),[TIME_STAMP],21) + '_' +
CAST(RATE_BID AS VARCHAR(10))),25,8) [CLOSE]
FROM dbo.TICKS
WHERE INSTRUMENT_ID = #InstrumentId AND TIME_STAMP BETWEEN #Start_Date AND #End_Date
GROUP BY DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime),
INSTRUMENT_ID
ORDER BY INTERVAL_TIME
PRINT CONVERT(NUMERIC(10,3),DATEDIFF(MS,#StartTime,GETDATE())/1000.)
SET #StartTime=GETDATE()
SELECT INSTRUMENT_ID,
DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime) INTERVAL_TIME,
CONVERT(INT,SUBSTRING(MIN(CONVERT(BINARY(8),[TIME_STAMP]) +
CAST(RATE_BID AS BINARY(4))),9,4)) [OPEN],
MAX(RATE_BID) HIGH,
MIN(RATE_BID) LOW,
CONVERT(INT,SUBSTRING(MAX(CONVERT(BINARY(8),[TIME_STAMP]) +
CAST(RATE_BID AS BINARY(4))),9,4)) [CLOSE]
FROM dbo.TICKS
WHERE INSTRUMENT_ID = #InstrumentId AND TIME_STAMP BETWEEN #Start_Date AND #End_Date
GROUP BY DATEADD(minute,(DATEDIFF(minute,#OffsetTime,[TIME_STAMP])/#Interval)*#Interval,#OffsetTime),
INSTRUMENT_ID
ORDER BY INTERVAL_TIME
PRINT CONVERT(NUMERIC(10,3),DATEDIFF(MS,#StartTime,GETDATE())/1000.)
On my system, I the string version executes in 490ms, and the binary version in 293ms.
I have the next stored procedure which inserts values into 2 tables. To the 2nd table I insert id's of 2 last inserts from 1st table
However, I would like to rewrite it with one query instead of using temp table and while.
CREATE PROCEDURE CurrencyExhange
AS
DECLARE #TmpTable Table
(
ID int IDENTITY(1,1) NOT NULL PRIMARY KEY,
BillID int,
Amount decimal,
Rate decimal,
Date date
)
INSERT INTO #TmpTable
SELECT T.[BillID]
,[Amount]
,CR.Rate
,CR.Date
FROM [FinanceLabkovich].[dbo].[Transactions] T
JOIN [FinanceLabkovich].[dbo].Bills B ON B.BillID = T.BillID
JOIN [FinanceLabkovich].[dbo].Currencies C ON C.CurrencyID=B.CurrencyID
JOIN [FinanceLabkovich].[dbo].CurrencyRates CR ON CR.CurrencyRateID=FinanceLabkovich.dbo.GetRate(T.Date)
WHERE LOWER(C.Name)='usd' AND T.Income=1
ORDER BY T.Date
DECLARE #ToBillID int = (SELECT BillID FROM [FinanceLabkovich].[dbo].Bills B WHERE B.Name='Purse')
DECLARE #i int = (SELECT MIN(Id) FROM #TmpTable)
DECLARE #maxId int = (SELECT MAX(Id) FROM #TmpTable)
DECLARE #TransactionID int, #ToTransactionID int, #Amount decimal
DECLARE #date date
WHILE (#i<=#maxId)
BEGIN
SET #date = (SELECT Date FROM #TmpTable WHERE ID=#i)
SET #Amount = (SELECT AmountUSD FROM [FinanceLabkovich].[dbo].Cashflow WHERE Date=#date)
IF #Amount > 0
BEGIN
INSERT INTO [FinanceLabkovich].[dbo].[Transactions] (Name,BillID,ToBillID,Amount,Date,Income)
SELECT "Name"='Currency exhange'
,BillID
,#ToBillID
,#Amount
,T.Date
,"Income"=0
FROM #TmpTable T
WHERE ID=#i
SET #TransactionID = ##IDENTITY
INSERT INTO [FinanceLabkovich].[dbo].[Transactions] (Name,BillID,ToBillID,Amount,Date,Income)
SELECT "Name"='Currency exhange'
,#ToBillID
,BillID
,#Amount*Rate AS Total
,Date
,"Income"=1
FROM #TmpTable WHERE ID=#i
SET #ToTransactionID = ##IDENTITY
INSERT INTO [FinanceLabkovich].[dbo].[Transfers]
SELECT #TransactionID, #ToTransactionID
END
SET #i += 1
END
Any help appreciated.
I have table 1 with 3 columns id, startdate and enddate. With order id being the primary key how do I list the dates between the date range Startdate and Enddate?
What I have:
id Startdate EndDate
1 2/11/2014 2/13/2014
2 2/15/2014 2/17/2014
What I need:
id Date
1 2/11/2014
1 2/12/2014
1 2/13/2014
2 2/15/2014
2 2/16/2014
2 2/17/2014
How do I do this?
Use recursive CTE:
WITH tmp AS (
SELECT id, StartDate AS [Date], EndDate
FROM MyTable
UNION ALL
SELECT tmp.id, DATEADD(DAY,1,tmp.[Date]), tmp.EndDate
FROM tmp
WHERE tmp.[Date] < tmp.EndDate
)
SELECT tmp.ID, tmp.[Date]
FROM tmp
ORDER BY tmp.id, tmp.[Date]
OPTION (MAXRECURSION 0) -- For long intervals
If you have to use cursor/loop, most times you are doing it wrong.
If you do a one-off setup of an auxiliary calendar table as shown at Why should I consider using an auxiliary calendar table?, possibly omitting a lot of the columns if you don't need them, like this:
CREATE TABLE dbo.Calendar
(
dt SMALLDATETIME NOT NULL
PRIMARY KEY CLUSTERED,
Y SMALLINT,
M TINYINT,
D TINYINT
)
GO
SET NOCOUNT ON
DECLARE #dt SMALLDATETIME
SET #dt = '20000101'
WHILE #dt < '20300101'
BEGIN
INSERT dbo.Calendar(dt) SELECT #dt
SET #dt = #dt + 1
END;
UPDATE dbo.Calendar SET
Y = YEAR(dt),
M = MONTH(dt),
D = DAY(dt);
(You may well not need the Y, M, D columns at all, but I left those in to show that more data can be stored for fast access - the article I linked to shows how that could be used.)
Then if your table is named "so", your code would simply be
SELECT A.id, C.dt
FROM so AS A
JOIN Calendar AS C
ON C.dt >= A.StartDate AND C.dt<= A.EndDate
An advantage of using an auxiliary table like that is that your queries can be faster: the work done in setting one up is a one-time cost which doesn't happen during usage..
Instead of using CTE (to over come recursive and performance when date range is large) below query can be used to get the list of dates between two date range.
DECLARE #StartDateSTR AS VARCHAR(32); DECLARE #EndDateSTR AS
VARCHAR(32); DECLARE #EndDate AS DATE; DECLARE #StartDate AS DATE;
SET #StartDateSTR = '01/01/1990'; SET #EndDateSTR = '03/31/2025'; SET
#StartDate = CAST(#StartDateSTR AS date); SET #EndDate =
cast(#EndDateSTR AS date); SELECT
DATEADD(DAY, n1.rn - 1, #StartDate) AS dt FROM (SELECT rn=Row_number() OVER( ORDER BY (SELECT NULL)) FROM sys.objects a
CROSS JOIN sys.objects b CROSS JOIN sys.objects c CROSS JOIN
sys.objects d) as n1 WHERE n1.[rn] <= Datediff(dd, #StartDate,
#EndDate)+1;
I have searched a lot, but cannot find a helpful answer:
i want to have a list of totals from a period the user defines by giving me a start and end date. The totals should every time being from the start date to beginning with the start date and add every row 1 day. so the last row gives the totals from start to end date.
example: - given period = start 2013-01-01 , end = 2013-01-31
total day 1 = 100
total day 2 = 0 (not listed in my totalsperday query, but should have a row in my final query)
total day 3 = 140
total day 4 = 20
...
final table should look like:
end day 1: 100
end day 2: 100
end day 3: 240
end day 4: 260
...
so i have a query who calculates all days:
SELECT '2013-01-01' as startdate, w.endDate
FROM
(
SELECT date('2013-01-01' + INTERVAL u.i*100 + v.i*10 + w.i DAY) AS endDate
FROM sysints AS u
JOIN sysints AS v
JOIN sysints AS w
WHERE ( u.i*100 + v.i*10 + w.i ) <=
(
SELECT DATEDIFF( '2013-01-31','2013-01-01') as ddff
)
) w
ORDER BY w.endDate ASC
and i have a query who calculates the totals per day
SELECT p.selldate, SUM(p.price) as totalPerDay
FROM products p
WHERE '2013-01-01' >= p.selldate <= '2013-01-31'
GROUP BY p.selldate
ORDER BY p.selldate ASC
now combining these two to get my final result is hard.
basically what the final query should look like is:
- make the sum of sumperday from day 1 to day 1
- make the sum of sumperday from day 1 to day 2
- make the sum of sumperday from day 1 to day 3
...
any help?
thx.
this is a simplified example of my final query.
Below is the sample. The idea is to obtain a initial data set ordered by date and having aggregate totals, implicit date range records. Then using the cursor you can pass through each row to get the final total column (amountCalc column in the sample) just by summarize the previous records - will work because you already have the columns ordered by date.
The procedure can have other input/ output parameters. Instead of getting info from table you can get data from one view, where the view can be already order by date asc. Is just a sample so can be customized as needed.
Good luck.
-- drop table `Balance`;
CREATE TABLE `Balance` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`date` DATE NOT NULL,
`account` varchar(30) NOT NULL,
`amount` DECIMAL(10,2) NOT NULL,
PRIMARY KEY (`id`)
);
INSERT INTO `Balance` (`date`, `account`, `amount`) VALUES
('2013-01-02', 'T355176', 8700),
('2013-01-03', 'T355176', 8900),
('2013-01-04', 'T355215', 33308),
('2013-01-03', 'T355215', 116581),
('2013-01-06', 'T812022', 275000),
('2013-01-02', 'T812063', 136500),
('2013-01-05', 'T812063', 11682),
('2013-01-06', 'T812064', 615100),
('2013-01-03', 'T812064', 25000),
('2013-01-02', 'T812085', 82500);
SELECT * FROM Balance WHERE date >= '2013-01-01' AND date <= '2013-01-06' ORDER BY date ASC;
CALL sp_getTotals('2013-01-01', '2013-01-06');
-- --------------------------------------------------------------------------------
-- Routine DDL
-- Note: comments before and after the routine body will not be stored by the server
-- --------------------------------------------------------------------------------
DELIMITER $$
CREATE DEFINER=`root`#`%` PROCEDURE `sp_getTotals`(IN startDate DATE, IN endDate DATE)
BEGIN
DECLARE dt DATE;
DECLARE amt DECIMAL(10,2);
DECLARE amtCalcPart DECIMAL(10,2);
DECLARE done INT DEFAULT 0;
DECLARE dtStart DATE;
DECLARE dtEnd DATE;
DECLARE cur1 CURSOR FOR SELECT date, amount FROM `TempMB`;
DECLARE cur2 CURSOR FOR SELECT startDate, endDate;
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
DROP TEMPORARY TABLE IF EXISTS `TempMB`;
CREATE TEMPORARY TABLE IF NOT EXISTS `TempMB` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`date` DATE NOT NULL,
`amount` DECIMAL(10,2) NULL DEFAULT 0.00,
`amountCalc` DECIMAL(10,2) NULL DEFAULT 0.00,
PRIMARY KEY (`id`)
);
SET dtStart = DATE(startDate);
SET dtEnd = DATE(endDate);
WHILE dtStart <= dtEnd DO
INSERT INTO `TempMB` (`date`) SELECT dtStart;
SET dtStart = DATE_ADD(dtStart, INTERVAL 1 DAY);
END WHILE;
SELECT * FROM TempMB;
-- Fill temp table with info needed
UPDATE `TempMB` t
INNER JOIN
(
SELECT date, SUM(amount) AS amount
FROM Balance
WHERE
date >= startDate AND date <= endDate
GROUP BY date
ORDER BY date ASC
) b ON b.date = t.date
SET
t.amount = b.amount;
/*INSERT INTO `TempMB` (`date`, `amount`)
SELECT date, SUM(amount) AS amount
FROM Balance
WHERE
date >= startDate AND date <= endDate
GROUP BY date
ORDER BY date ASC;
*/
SET amtCalcPart = 0.00;
-- Initialise cursor
OPEN cur1;
-- USE BEGIN-END handler for cursor-control within own BEGIN-END block
BEGIN
DECLARE EXIT HANDLER FOR NOT FOUND BEGIN END;
-- Loop cursor throu temp records
LOOP
-- Get next value
FETCH cur1 INTO dt, amt;
-- Calculate amountCalc
SET amtCalcPart = (SELECT SUM(amount) as amt FROM `TempMB` WHERE Date <= dt);
UPDATE `TempMB` SET amountCalc = amtCalcPart WHERE date = dt;
END LOOP;
END;
-- Release cursor
CLOSE cur1;
SELECT * FROM TempMB;
END