SELECT
gst_journal.*,
salesmain.TransactionNo
FROM
`gst_journal`
LEFT JOIN salesmain
ON (
gst_journal.docNo = salesmain.InvoiceNo
AND gst_journal.Source = salesmain.Source
)
WHERE (
MONTH(salesmain.TDate) = '10'
AND YEAR(salesmain.TDate) = '2013'
AND salesmain.TDate BETWEEN STR_TO_DATE('2013-10-01', '%Y-%m-%d')
AND STR_TO_DATE('2013-10-31', '%Y-%m-%d')
AND (
(
gst_journal.inOutTax = 'OUT'
AND gst_journal.taxType IN ('SR', 'GST')
AND gst_journal.Source IN ('AR', 'NR', 'BR', 'PR', 'JV', 'CR')
)
)
)
OR (
MONTH(gst_journal.TDate) = '10'
AND YEAR(gst_journal.TDate) = '2013'
AND (
gst_journal.TDate BETWEEN STR_TO_DATE('2013-10-01', '%Y-%m-%d')
AND STR_TO_DATE('2013-10-31', '%Y-%m-%d')
AND (
gst_journal.taxType = 'SR'
AND gst_journal.Source IN ('BR')
)
)
)
ORDER BY gst_journal.Tdate,
salesmain.TDate ;
this query run properly and records give correct but it take to time run.
Any one help to optimize time query.
Thanks
Related
I'm working on a big query in SQL (MySQL 5.7) to calculate aggregated columns based on raw values in my table. I've created several aggregated columns (see attached screenshot and SQL) and I now need to create a conversion_percent column for each tlp_aff_id in my query.
This conversion_percent should be a division of the aggregated JoinedSessions.total_sessions and COUNT(Report.tlp_aff_id) as leads_total.
My current SQL:
SELECT
# Application details
Report.tlp_aff_id,
# Revenue
JoinedRevenue.total_revenue,
# Commission
JoinedCommission.total_commission,
# Profit
JoinedProfit.total_profit,
# Sessions
JoinedSessions.total_sessions,
# Submits
COUNT(Report.tlp_aff_id) as total_submits,
# Leads
COUNT(Report.tlp_aff_id) as leads_total,
SUM(case when Report.application_result = 'Accepted' then 1 else 0 end) leads_accepted,
SUM(case when Report.application_result = 'Rejected' then 1 else 0 end) leads_rejected
# Conversion percent
# JoinedConversion.conversion_percent
FROM
tlp_payout_report_minute AS Report
INNER JOIN
(
SELECT
JoinedRevenue.tlp_aff_id,
JoinedRevenue.minute_rounded_timestamp,
SUM(commission) AS total_revenue
FROM
tlp_payout_report_minute AS JoinedRevenue
WHERE
JoinedRevenue.minute_rounded_timestamp >= 1664841600
AND
JoinedRevenue.minute_rounded_timestamp <= 1664927999
GROUP BY
JoinedRevenue.tlp_aff_id
) AS JoinedRevenue
ON JoinedRevenue.tlp_aff_id = Report.tlp_aff_id
INNER JOIN
(
SELECT
ReportCommission.tlp_aff_id,
ReportCommission.seller_code,
ReportCommission.minute_rounded_timestamp,
SUM(commission) AS total_commission
FROM
tlp_payout_report_minute AS ReportCommission
WHERE
ReportCommission.minute_rounded_timestamp >= 1664841600
AND
ReportCommission.minute_rounded_timestamp <= 1664927999
AND
ReportCommission.seller_code != 44
GROUP BY
ReportCommission.tlp_aff_id
) AS JoinedCommission
ON JoinedCommission.tlp_aff_id = Report.tlp_aff_id
INNER JOIN
(
SELECT
ReportProfit.tlp_aff_id,
ReportProfit.seller_code,
ReportProfit.application_result,
ReportProfit.minute_rounded_timestamp,
SUM(commission) AS total_profit
FROM
tlp_payout_report_minute AS ReportProfit
WHERE
ReportProfit.minute_rounded_timestamp >= 1664841600
AND
ReportProfit.minute_rounded_timestamp <= 1664927999
AND
ReportProfit.application_result = 'Accepted'
AND
ReportProfit.seller_code = 44
GROUP BY
ReportProfit.tlp_aff_id
) AS JoinedProfit
ON JoinedProfit.tlp_aff_id = Report.tlp_aff_id
INNER JOIN
(
SELECT
Conversion.aff_id,
Conversion.conversion_type,
COUNT(Conversion.ip_address) as total_sessions
FROM
tlp_conversions AS Conversion
WHERE
Conversion.conversion_time >= '2022-10-04 00:00:00'
AND
Conversion.conversion_time <= '2022-10-04 23:59:59'
AND
Conversion.aff_id IS NOT NULL
AND
Conversion.conversion_type = 2
GROUP BY
Conversion.aff_id
) AS JoinedSessions
ON JoinedSessions.aff_id = Report.tlp_aff_id
WHERE
Report.minute_rounded_timestamp >= 1664841600
AND
Report.minute_rounded_timestamp <= 1664927999
GROUP BY
Report.tlp_aff_id
ORDER BY
JoinedRevenue.total_revenue DESC
I'm thinking something along the lines of:
INNER JOIN
(
...
) AS JoinedConversion
ON JoinedConversion.aff_id = Report.tlp_aff_id
But I don't think this is necessary for conversion_percent.
What's the right approach here?
I am stuck at a point where I have millions of records and required different joins for the same. Also there are some tricky part for the order clause. My query performs fast result if I'll not apply any ordering. But while applying order clause it takes too much time to get result.
Without Order clause it results in 5-6 seconds.
While applying Order Clause it results in 40-45 second
SELECT
forms_values.id,
CASE
WHEN forms_values.appointment_type = 2
AND user_patient_assinged_to_doctor.start_time IS NOT NULL
THEN
CASE
WHEN patient_responded_tags_logs.tag_set_at IS NOT NULL
THEN
CASE
WHEN UNIX_TIMESTAMP(
CONVERT_TZ(
patient_responded_tags_logs.tag_set_at,
"+00:00",
"-06:00"
)
) > UNIX_TIMESTAMP(
CONVERT_TZ(
STR_TO_DATE(
CONCAT(
user_patient_assinged_to_doctor.date,
" ",
user_patient_assinged_to_doctor.start_time
),
"%Y-%m-%d %h:%i %p"
),
"+00:00",
"-06:00"
)
)
THEN UNIX_TIMESTAMP(
CONVERT_TZ(
patient_responded_tags_logs.tag_set_at,
"+00:00",
"-06:00"
)
)
ELSE UNIX_TIMESTAMP(
CONVERT_TZ(
STR_TO_DATE(
CONCAT(
user_patient_assinged_to_doctor.date,
" ",
user_patient_assinged_to_doctor.start_time
),
"%Y-%m-%d %h:%i %p"
),
"+00:00",
"-06:00"
)
)
END
ELSE UNIX_TIMESTAMP(
CONVERT_TZ(
STR_TO_DATE(
CONCAT(
user_patient_assinged_to_doctor.date,
" ",
user_patient_assinged_to_doctor.start_time
),
"%Y-%m-%d %h:%i %p"
),
"+00:00",
"-06:00"
)
)
END
ELSE
CASE
WHEN patient_responded_tags_logs.tag_set_at IS NOT NULL
THEN
CASE
WHEN UNIX_TIMESTAMP(
CONVERT_TZ(
patient_responded_tags_logs.tag_set_at,
"+00:00",
"-06:00"
)
) > UNIX_TIMESTAMP(forms_values.created_at)
THEN UNIX_TIMESTAMP(
CONVERT_TZ(
patient_responded_tags_logs.tag_set_at,
"+00:00",
"-06:00"
)
)
ELSE UNIX_TIMESTAMP(forms_values.created_at)
END
ELSE UNIX_TIMESTAMP(forms_values.created_at)
END
END AS "consultation_date_time_ordering",
CASE
WHEN forms_values.appointment_type = 2
AND user_patient_assinged_to_doctor.start_time IS NOT NULL
THEN UNIX_TIMESTAMP(
CONVERT_TZ(
STR_TO_DATE(
CONCAT(
user_patient_assinged_to_doctor.date,
" ",
user_patient_assinged_to_doctor.start_time
),
"%Y-%m-%d %h:%i %p"
),
"+00:00",
"-06:00"
)
)
ELSE UNIX_TIMESTAMP(forms_values.created_at)
END AS "consultation_date_time" ,
CASE
WHEN forms_values.is_postpone = '1'
OR forms_values.is_completed = '8'
THEN
CASE
WHEN UNIX_TIMESTAMP(
CONVERT_TZ(
STR_TO_DATE(
CONCAT(UTC_DATE(), ' ', UTC_TIME()),
'%Y-%m-%d %h:%i:%s'
),
'+00:00',
'-06:00'
)
) < UNIX_TIMESTAMP(
my_list_postpone.postponed_date
)
THEN 0
ELSE 1
END
ELSE 1
END AS "postponed_consultation_ordering"
FROM
`forms_values`
LEFT JOIN `forms_values_completed_status_details`
ON `forms_values_completed_status_details`.`form_value_id` = `forms_values`.`id`
/*INNER JOIN `users`
ON `users`.`id` = `forms_values`.`patient_id`
LEFT JOIN `users` AS `doctors`
ON `doctors`.`id` = `forms_values`.`doctor_id`*/
LEFT JOIN `user_patient_assinged_to_doctor`
ON `user_patient_assinged_to_doctor`.`form_value_id` = `forms_values`.`id`
INNER JOIN `states_countries`
ON `forms_values`.`state` = `states_countries`.`id`
LEFT JOIN `user_payment_history`
ON `user_payment_history`.`form_value_id` = `forms_values`.`id`
LEFT JOIN `emailed_tags_logs`
ON `emailed_tags_logs`.`form_value_id` = `forms_values`.`id`
AND `emailed_tags_logs`.`id` =
(SELECT
emailed_tags_logs.id
FROM
emailed_tags_logs
WHERE emailed_tags_logs.form_value_id = forms_values.id
AND emailed_tags_logs.id =
(SELECT
emailed_tags_logs1.id AS emtid
FROM
emailed_tags_logs AS emailed_tags_logs1
WHERE emailed_tags_logs1.form_value_id = forms_values.id
ORDER BY emailed_tags_logs1.created_at DESC
LIMIT 1)
AND emailed_tags_logs.status IN (1, 3)
AND emailed_tags_logs.is_pt_responded = "0"
ORDER BY emailed_tags_logs.created_at DESC
LIMIT 1)
LEFT JOIN `my_list_assign_doctor`
ON `my_list_assign_doctor`.`form_value_id` = `forms_values`.`id`
AND `my_list_assign_doctor`.`id` =
(SELECT
my_list_assign_doctor.id
FROM
my_list_assign_doctor
WHERE my_list_assign_doctor.form_value_id = forms_values.id
AND my_list_assign_doctor.status IN (1, 2)
AND my_list_assign_doctor.prior_type = "others"
ORDER BY my_list_assign_doctor.created_at DESC
LIMIT 1)
LEFT JOIN `my_list_assign_doctor` AS `my_list_postpone`
ON `my_list_postpone`.`form_value_id` = `forms_values`.`id`
AND `forms_values`.`is_postpone` IN ('1', '2')
AND `my_list_postpone`.`id` =
(SELECT
my_list_assign_doctor.id
FROM
my_list_assign_doctor
WHERE my_list_assign_doctor.form_value_id = forms_values.id
AND my_list_assign_doctor.prior_type = "postpone"
ORDER BY my_list_assign_doctor.created_at DESC
LIMIT 1)
LEFT JOIN `users` AS `partner`
ON `user_payment_history`.`std_partner_id` = `partner`.`id`
LEFT JOIN `patient_responded_tags_logs`
ON `patient_responded_tags_logs`.`form_value_id` = `forms_values`.`id`
AND `patient_responded_tags_logs`.`status` = '1'
LEFT JOIN `user_subscriptions`
ON `user_subscriptions`.`user_payment_history_id` = `user_payment_history`.`id`
AND `user_payment_history`.`form_value_id` = `forms_values`.`id`
ORDER BY
postponed_consultation_ordering DESC,
`consultation_date_time` DESC
LIMIT 10 OFFSET 0
Note: All joins are important, and fields are removed from select query for some concerns.
Explain summary
The problem is the combination of LIMIT and ORDER BY.
Without the ORDER BY the query will stop as soon as the first ten random rows are selected. With the order by the query must gather all possible rows then sort them in order then only return the first 10.
There is really no way to make this any faster given that 'postponed_consultation_datetime' is a calculated field.
Some of these composite indexes may be helpful.
user_patient_assinged_to_doctor: INDEX(form_value_id, start_time, date)
patient_responded_tags_logs: INDEX(form_value_id, tag_set_at, status)
my_list_postpone: INDEX(form_value_id, postponed_date, id)
forms_values_completed_status_details: INDEX(form_value_id)
user_payment_history: INDEX(form_value_id, std_partner_id, id)
user_subscriptions: INDEX(user_payment_history_id)
emailed_tags_logs: INDEX(form_value_id, created_at, id)
emailed_tags_logs: INDEX(form_value_id, id, status, is_pt_responded, created_at)
my_list_assign_doctor: INDEX(form_value_id, status, prior_type, created_at, id)
my_list_assign_doctor: INDEX(form_value_id, prior_type, created_at, id)
my_list_assign_doctor: INDEX(form_value_id, id, status, prior_type, created_at)
Don't use LEFT JOIN when you mean INNER JOIN.
FALSE is 0; TRUE is anything else. See if you can simplify some of the CASE statements:
CASE WHEN boolean THEN 1 ELSE 0
is identical to
boolean
Similarly:
CASE WHEN boolean THEN 0 ELSE 1
can probably be replaced by
With proper use of TIMESTAMP versus DATETIME, you can possibly get rid of all the calls to CONVERT_TZ.
NOT boolean
However, this may help the most... Turn the query inside out. See what the minimal effort is to find the LIMIT 10 ids. Hopefully, you only need to look at one, or a small number of, the tables. Then use that as a subquery ("derived table") and then reach for the rest of the columns.
I'm trying to optimize a query, which as the db is growing, its performance is severely lacking.
Background: we are trying to find a list of users who have taken a course and their credential is now due to be renewed (or has not renewed). In searching the query we have to have a look into the registration table (which is the same table that holds all their registration history) and find records where they have not renewed. (Each time the client takes a course they have a registration record added.) The query I'm wanting to optimize looks to see if they've (client) taken the same course type on a date/time after the last class (of same type) they took. If there is no record it should result row(s) that they didn't renew their course. it sounds easy, but, as you know, when you're in the heat of writing a query it gets very complex--and even more so once the db has grown to be so large that it takes almost 5-6 minutes to find the data. So, I'm asking for help on how I can optimize the efforts of my predecessor, below.
Here is the query, thus far (don't laugh, it wasn't started by me--I took over the project).
I have no clue where to begin with optimizing this MySQL. I think it needs to have select statements within the JOINS, but I'm at your mercy to direct me as to where to start! (I"m not a db guy, but offered to take a look and see where we can fix this).
Thanks a million for reading.
Lee
SELECT
r.GUID AS `A/C #`,
concat( a.AttendeeLastName, ', ', a.AttendeeFirstName ) AS Full Name (Last, First),
r.CourseExpirationDateFull AS `Exp Date`,
mtype_master_abbrev AS Course,
a.EmailName AS Email,
r.EventID,
r.EventTypeMasterID,
m.type_master_name,
IF( ( r.CourseExpirationDateFull < curdate( ) ), 'Expired', 'Valid' ) AS Status,
e.StartDateTime,
( to_days( curdate( ) ) - to_days( r.ExpNoticeSent ) ) AS Last Notice,
r.AttendeeID,
a.AttendeeCredentials,
r.RegistrationID,
r.RenewedExternalYYYY,
r.ExpNoticeSent,
q.RenewedRegID,
rs.reg_status_name AS `Reg Status`,
( to_days( r.CourseExpirationDateFull ) - to_days( curdate( ) ) ) AS Days2Exp,
a.flgReturnEmail,
a.flgSendEmail,
r.reg_type_ID,
a._usr_flg_do_not_call,
a.flgPrintLetter
e.EventTypeMasterID AS MasterID,
c.Last: yy-mm-dd - by - topic AS LastComm,
r.reg_renewal_status_id
FROM
vjgzuqrr_wtsql.registration r LEFT JOIN vjgzuqrr_wtsql.events ON ( r.EventID = events.EventID ) LEFT JOIN
vjgzuqrr_wtsql.attendees a ON a.ID = r.GUID LEFT JOIN
vjgzuqrr_wtsql.tbl_crs_type_master m ON r.EventTypeMasterID = m.ID_crs_type_master LEFT JOIN
vjgzuqrr_wtsql.qryrenreg q ON r.RegistrationID = q.OrigRegID LEFT JOIN
vjgzuqrr_wtsql.tbl_reg_status rs ON rs.ID_reg_status = r.RegistrationStatus LEFT JOIN
vjgzuqrr_wtsql.v_last_contact c ON c.registrationid = r.RegistrationID
WHERE
r.Role = 1
AND r.reg_type_ID IN ( 1, 2 )
AND r.CompletionStatus IN ( 9, 8 )
AND r.r IN ( 1, 14, 9 )
AND ( r.EventTypeMasterID IS NOT NULL OR r.EventTypeMasterID = 17 )
AND r.flgDelete = 0
AND r.flgTest = 0
AND e.flgDelete = 0
AND e.flgTestCourse = 0
AND e.flgDelete = 0
AND a.flgTest = 0
AND isnull( q.RenewedRegID )
AND a.flgReturnEmail = 0
AND m.type_master_abbrev NOT IN ( 'EKGPHARM', 'IVCERT', 'sem', 'fam&friends', 'cccc' )
Edit to include Explain:
Sorry im a bit slow, mysql,
This does not speed anything up ( i think, but it may help a bit), but it should help in reading it in a non-mindbreaking way. (hopefully this will also help others look at it.)
SELECT
r.GUID AS `A/C #`,
concat( a.AttendeeLastName, ', ', a.AttendeeFirstName ) AS Full Name (Last, First),
r.CourseExpirationDateFull AS `Exp Date`,
mtype_master_abbrev AS Course,
a.EmailName AS Email,
r.EventID,
r.EventTypeMasterID,
m.type_master_name,
IF( ( r.CourseExpirationDateFull < curdate( ) ), 'Expired', 'Valid' ) AS Status,
e.StartDateTime,
( to_days( curdate( ) ) - to_days( r.ExpNoticeSent ) ) AS Last Notice,
r.AttendeeID,
a.AttendeeCredentials,
r.RegistrationID,
r.RenewedExternalYYYY,
r.ExpNoticeSent,
q.RenewedRegID,
rs.reg_status_name AS `Reg Status`,
( to_days( r.CourseExpirationDateFull ) - to_days( curdate( ) ) ) AS Days2Exp,
a.flgReturnEmail,
a.flgSendEmail,
r.reg_type_ID,
a._usr_flg_do_not_call,
a.flgPrintLetter
e.EventTypeMasterID AS MasterID,
c.Last: yy-mm-dd - by - topic AS LastComm,
r.reg_renewal_status_id
FROM
vjgzuqrr_wtsql.registration r LEFT JOIN
vjgzuqrr_wtsql.events e ON r.EventID = e.EventID LEFT JOIN
vjgzuqrr_wtsql.attendees a ON a.ID = r.GUID LEFT JOIN
vjgzuqrr_wtsql.tbl_crs_type_master m ON r.EventTypeMasterID = m.ID_crs_type_master LEFT JOIN
vjgzuqrr_wtsql.qryrenreg q ON r.RegistrationID = q.OrigRegID LEFT JOIN
vjgzuqrr_wtsql.tbl_reg_status rs ON rs.ID_reg_status = r.RegistrationStatus LEFT JOIN
vjgzuqrr_wtsql.v_last_contact c ON c.registrationid = r.RegistrationID
WHERE
r.Role = 1
AND r.reg_type_ID IN ( 1, 2 )
AND r.CompletionStatus IN ( 9, 8 )
AND r.r IN ( 1, 14, 9 )
AND ( r.EventTypeMasterID IS NOT NULL OR r.EventTypeMasterID = 17 )
AND r.flgDelete = 0
AND r.flgTest = 0
AND e.flgDelete = 0
AND e.flgTestCourse = 0
AND e.flgDelete = 0
AND a.flgTest = 0
AND isnull( q.RenewedRegID )
AND a.flgReturnEmail = 0
AND m.type_master_abbrev NOT IN ( 'EKGPHARM', 'IVCERT', 'sem', 'fam&friends', 'cccc' )
I have the following MySQL query
select a.*, d.*, p.*, pow.* from appointment a
left join doctor d on d.id = a.doctor_id
left join patient p on p.id = a.patient_id
left join point_of_work pow on pow.id = a.point_of_work_id
where (doctor_id, patient_id, date) = (
select doctor_id, patient_id,
coalesce(
min(case when date > curdate() then date end),
max(case when date < curdate() then date end)
) date
from appointment
where (doctor_id, patient_id) = (a.doctor_id, a.patient_id)
)
and d.external_id = 1
And I am trying to convert it to DQL.
Right now I come to this version of DQL but it seems I'am doing something (or maybe more things:( wrong)
$expr = $this->getEntityManager()->getExpressionBuilder();
$queryBuilder = $this->createQueryBuilder('a')
->leftJoin(Doctor::class, 'd')
->leftJoin(Patient::class, 'p')
->leftJoin(PointOfWork::class, 'pow')
->where(
$expr->eq('doctorId, patient_id, date',
$this->createQueryBuilder('a')
->select(Appointment::class . ',
coalesce(
min(case when date > curdate() then date end),
max(case when date < curdate() then date end)
) date'
)
->where ('(doctorId, patientId) = (a.doctorId, a.patientId)')
)
)
->andWhere('d.externalId = :externalId')
->setParameter('externalId', $doctorExternalId)
->setMaxResults($limit)
->setFirstResult($offset);
What approaches do I need for the DQL conversion?
I get error
SQL Error (1055): Expression #7 of SELECT list is not in GROUP BY clause and contains nonaggregated column 'ifu.amount' which is not functionally dependent on columns in GROUP BY clause; this is incompatible with sql_mode=only_full_group_by
after migrating to mysql 8.0 from 5.6. I know that it can be easily fixed by disabling ONLY_FULL_GROUP_BY flag, but I want it to be more compatible with mysql 8.0. So question is if I would add ifu.amount to GROUP BY it should work perfetcly fine and I won't miss any query results or anything? Now without GROUP BY ifu.amount MySQL code looks like:
select
`i`.`id` AS `institution_id`,
`i`.`name` AS `institution_name`,
`cr`.`check_date` AS `check_date`,
sum(
(
case when (`cr`.`status` = '1') then 1 else 0 end
)
) AS `can_accept`,
sum(
(
case when (`cr`.`status` = '0') then 1 else 0 end
)
) AS `cannot_accept`,(
sum(
(
case when (`cr`.`status` = '1') then 1 else 0 end
)
) + sum(
(
case when (`cr`.`status` = '0') then 1 else 0 end
)
)
) AS `suma`,
`ifu`.`amount` AS `amount`,
round(
(
(
(
(
sum(
(
case when (`cr`.`status` = '1') then 1 else 0 end
)
) * 100
) / (
sum(
(
case when (`cr`.`status` = '1') then 1 else 0 end
)
) + sum(
(
case when (`cr`.`status` = '0') then 1 else 0 end
)
)
)
) * `ifu`.`amount`
) * 0.01
),
2
) AS `financed_amount`
from
(
(
(
`check_results` `cr`
join `family_doctors` `fd` on((`fd`.`id` = `cr`.`doctor_id`))
)
join `institutions` `i` on((`i`.`id` = `fd`.`institution_id`))
)
join `institutions_funding` `ifu` on((`ifu`.`institution_id` = `i`.`id`))
)
where
(`cr`.`status` in (1, 0))
group by
`i`.`id`,
`i`.`name`,
`cr`.`check_date`
Thanks for help in advance!
Include amount in your group by clause.
where
(`cr`.`status` in (1, 0))
group by
`i`.`id`,
`i`.`name`,
`cr`.`check_date`,
`ifu`.`amount`
if amount is excluded on your group by clause, this will get the amount that corresponds on your id, name and check date in ascending order (default).
or
min(`ifu`.`amount`) as `amount`.