Time for import from XML to MySQL - mysql
I use a script to import an XML file into an MySQL database. The XML contains around 165.000 datasets.
At the moment I have imported 25.000 datasets - and this has takes now around 70 mins. Is that normal?
The import runs from shell on an dedicated server with following specs:
Intel(R) Xeon(R) CPU E5-1620 v2 # 3.70GHz
8 Cores, 10240KB Cache
RAM 64GB
load average: 3.53, 3.06, 2.80
I can't believe that should be normal?
EDIT (add more info)
MySQL v 5.5.50
XML file:
<?xml version="1.0" encoding="utf-8" ?>
<amateures>
<amateur>
<id></id>
<login></login>
<figur></figur>
<vote_rating></vote_rating>
<haarfarbe></haarfarbe>
<gewicht></gewicht>
<groesse></groesse>
<plz></plz>
<sternzeichen></sternzeichen>
<vorlieben></vorlieben>
<vorlieben_en></vorlieben_en>
<uebermich></uebermich>
<uebermich_en></uebermich_en>
<geschlecht></geschlecht>
<staat></staat>
<gebtag></gebtag>
<regdate></regdate>
<urls>
<profil></profil>
<register></register>
</urls>
<images>
<overview></overview>
<overview_fsk16></overview_fsk16>
<overview_big></overview_big>
<overview_fsk16_big></overview_fsk16_big>
</images>
<videos>
<video>
<title></title>
<title_en></title_en>
<beschreibung></beschreibung>
<beschreibung_en></beschreibung_en>
<playtime></playtime>
<id></id>
<img_fsk16></img_fsk16>
<img></img>
<imgani></imgani>
<videourl></videourl>
<category></category>
[...]
<category></category>
</video>
<video>
[...]
</video>
</videos>
</amateur>
</amateures>
and php file:
#!/usr/bin/php -n
<?php
function OpenDB () {
$connect = mysqli_connect('localhost','','');
if(!$connect) die ("Connection to SQL-Server failed!");
$database = mysqli_select_db($connect, "");
if(!$database) die ("Selection of Database failed!");
return $connect;
}
$connect = OpenDB();
$count = 0;
$countnew = 0;
$countvid = 0;
$starttime = time();
$xml = simplexml_load_file("import.xml");
$query = "update amateur set deleted = 1";
mysqli_query($connect, $query);
$query = "delete from amateur_vid";
$res = mysqli_query($connect, $query);
foreach ($xml->amateur as $amateur) {
$count++;
$amateur_id = $amateur->id;
$amateur_login = utf8_decode($amateur->login);
$amateur_figur = $amateur->figur;
$amateur_vote_rating = $amateur->vote_rating;
$amateur_haarfarbe = $amateur->haarfarbe;
$amateur_gewicht = $amateur->gewicht;
$amateur_groesse = $amateur->groesse;
$amateur_plz = $amateur->plz;
$amateur_sternzeichen = $amateur->sternzeichen;
$amateur_vorlieben = $amateur->vorlieben;
$amateur_vorlieben_en = $amateur->vorlieben_en;
$amateur_uebermich = $amateur->uebermich;
$amateur_uebermich_en = $amateur->uebermich_en;
$amateur_geschlecht = $amateur->geschlecht;
$amateur_staat = $amateur->staat;
$amateur_gebtag = $amateur->gebtag;
$amateur_regdate = $amateur->regdate;
foreach ($amateur->urls as $url) {
$amateur_profil = $url->profil;
$amateur_register = $url->register;
}
foreach ($amateur->images as $image) {
$amateur_overview = $image->overview;
$amateur_overview_fsk16 = $image->overview_fsk16;
$amateur_overview_big = $image->overview_big;
$amateur_overview_fsk16_big = $image->overview_fsk16_big;
}
$amateur_vorlieben = mysqli_escape_string($connect, $amateur_vorlieben);
$amateur_vorlieben_en = mysqli_escape_string($connect, $amateur_vorlieben_en);
$amateur_uebermich = mysqli_escape_string($connect, $amateur_uebermich);
$amateur_uebermich_en = mysqli_escape_string($connect, $amateur_uebermich_en);
$query = "select * from amateur where amateur_id = $amateur_id";
$res = mysqli_query($connect, $query);
if(mysqli_num_rows($res) == 0) {
$countnew++;
$query = "insert into amateur (amateur_id, amateur_login, amateur_figur, amateur_vote_rating, amateur_haarfarbe, amateur_gewicht, amateur_groesse, amateur_plz, amateur_sternzeichen, amateur_vorlieben, amateur_vorlieben_en, amateur_uebermich, amateur_uebermich_en, amateur_geschlecht, amateur_staat, amateur_gebtag, amateur_regdate, amateur_profil, amateur_register, amateur_overview, amateur_overview_fsk16, amateur_overview_big, amateur_overview_fsk16_big) values ($amateur_id, '$amateur_login', '$amateur_figur', '$amateur_vote_rating', '$amateur_haarfarbe', '$amateur_gewicht', '$amateur_groesse', '$amateur_plz', '$amateur_sternzeichen', '$amateur_vorlieben', '$amateur_vorlieben_en', '$amateur_uebermich', '$amateur_uebermich_en', '$amateur_geschlecht', '$amateur_staat', '$amateur_gebtag', '$amateur_regdate', '$amateur_profil', '$amateur_register', '$amateur_overview', '$amateur_overview_fsk16', '$amateur_overview_big', '$amateur_overview_fsk16_big')";
$res = mysqli_query($connect, $query);
} else {
$query = "update amateur set amateur_login='$amateur_login', amateur_figur='$amateur_figur', amateur_vote_rating='$amateur_vote_rating', amateur_haarfarbe='$amateur_haarfarbe', amateur_gewicht='$amateur_gewicht', amateur_groesse='$amateur_groesse', amateur_plz='$amateur_plz', amateur_sternzeichen='$amateur_sternzeichen', amateur_vorlieben='$amateur_vorlieben', amateur_vorlieben_en='$amateur_vorlieben_en', amateur_uebermich='$amateur_uebermich', amateur_uebermich_en='$amateur_uebermich_en', amateur_geschlecht='$amateur_geschlecht', amateur_staat='$amateur_staat', amateur_gebtag='$amateur_gebtag', amateur_regdate='$amateur_regdate', amateur_profil='$amateur_profil', amateur_register='$amateur_register', amateur_overview='$amateur_overview', amateur_overview_fsk16='$amateur_overview_fsk16', amateur_overview_big='$amateur_overview_big', amateur_overview_fsk16_big='$amateur_overview_fsk16_big', deleted=0 where amateur_id = $amateur_id";
$res = mysqli_query($connect, $query);
if(!$res)
die($query);
}
foreach ($amateur->videos as $videos) {
foreach ($videos->video as $video) {
$vids_title = $video->title;
$vids_title_en = $video->title_en;
$vids_beschreibung = $video->beschreibung;
$vids_beschreibung_en = $video->beschreibung_en;
$vids_playtime = $video->playtime;
$vids_id = $video->id;
$vids_img_fsk16 = $video->img_fsk16;
$vids_img = $video->img;
$vids_imgani = $video->imgani;
$vids_videourl = $video->videourl;
$vids_category = $video->category;
$query = "insert into amateur_vid (amateur_id, vids_title, vids_title_en, vids_beschreibung, vids_beschreibung_en, vids_playtime, vids_id, vids_img_fsk16, vids_img, vids_imgani, vids_videourl, vids_category) values ($amateur_id, '$vids_title', '$vids_title_en', '$vids_beschreibung', '$vids_beschreibung_en', '$vids_playtime', '$vids_id', '$vids_img_fsk16', '$vids_img', '$vids_imgani', '$vids_videourl', '$vids_category')";
$res = mysqli_query($connect, $query);
$setid = mysqli_insert_id($connect);
$countvid++;
}
}
}
$endtime = time();
$interval = $endtime-$starttime;
$query = "select count(id) from amateur where deleted = 1";
$res = mysqli_query($connect, $query);
$countdel = mysqli_result($res,0,0);
$query = "delete from amateur where deleted = 1";
mysqli_query($connect, $query);
?>
My PHP script has a small issue. I cannot import all categories - it will be only import one category each video.
I start already a question for that.
Hope this informations enough? This script was writter by an programmer for me many years before for another XML import. I adopt and change it for this new XML file.
I'm not a professional programmer and not an native English speaker - so please talk with "easy" words to me ;)
Related
Why does this not work? MySQL (TRYING TO MAKE A WHITELIST UPDATE)
Trying to make a whitelist system with the arguments below, but It is not echoing anything. The code is not function as it should atm. The problem is that it is not updating it. This is the code:
$servername = "localhost";
$username = "root";
$password = "";
$dbname = "premium";
$key = $_GET["key"];
$user = $_GET["user"];
$id = $_GET["id"];
// Create connection
$conn = new mysqli($servername, $username, $password, $dbname);
// Check connection
if ($conn->connect_error) {
die("Connection failed: " . $conn->connect_error);
}
$result = mysqli_query("SELECT * FROM `whitelists` WHERE `key` = '$key'");
if( $result === false ) {
echo 'nope';
}
else
{
echo 'yes';
mysqli_query("UPDATE `whitelists` SET `id` = '$id', `user` = '$user' WHERE `key` ='$key' ");
}
$conn->close();
?>
first thing you should do is Use mysqli_* functions or PDO, because your code is Deprecated.
example:
$result = mysqli_query($conn, "UPDATE whitelists SET `id` = '$id', `user` = '$user' WHERE `key` ='$key' ");
Secondly you should use a false statement instead of num_rows.
if($result === false)
{
echo "nope";
}
else
{
// Execute your query here.
}
How can i get mysql_result() to work
I am new to programming, I've been trying to create a hit_counter for a demo site I am developing but I keep getting an error for the MySQL_result() function.
$user_ip = $_SERVER['REMOTE_ADDR'];
function ip_exists($ip) {
include 'include.php';
global $user_ip;
$query = "SELECT `ip` FROM `ip` WHERE`ip`= '$user_ip'";
$mysql_run = mysqli_query($con, $query);
$query_num_rows = mysqli_num_rows($mysql_run);
if ($query_num_rows == 0) {
return false;
} else if ($query_num_rows >= 1) {
return true;
}
}
function ip_add($ip) {
include 'include.php';
$query = "INSERT INTO `ip` VALUES('$ip')";
$query_run = mysqli_query($con, $query);
}
function update_count() {
include 'include.php';
$query = "SELECT `hits` FROM `hit_count`";
if ($query_run = mysqli_query($con, $query)) {
$count = mysql_result($query_run, 0, 'hits');
$count_inc = $count + 1;
$query_update = "UPDATE ``hit_count` SET `hits`= '$count_inc'";
$query_update_run = mysqli_query($con, $query_update);
}
}
if (!ip_exists($user_ip)) {
update_count();
ip_add($user_ip);
}
You are mixing mysql_*() and mysqli_*() functions; they belong to different MySQL extensions and cannot be interchanged. Since the rest of your code uses mysqli, make sure you don't use any mysql_*() function. The old MySQL PHP extension (that provides the mysql_*() functions) has been deprecated in PHP 5.5 and completely removed in PHP 7. Using MySQLi, the equivalent of $count = mysql_result($query_run, 0, 'hits') is: $row = mysqli_fetch_assoc($query_run); $count = $row['hits'];
overwrite existing data based on returned values
Trying to overwrite existing data based on returned values, just want to get 'location' to update to the new info coming from a CSV import. Have a duplicated table with Oldlocation (all set to DW for testing, so any changes can be seen) Seems like it should be simple....:
$file = "allinv.CSV";
if(($handle = fopen($file, "r")) !== FALSE)
{ fgetcsv($handle);
while (($data = fgetcsv($handle, 1000, ",")) !== FALSE){
$num = count($data);
for ($c=0; $c < $num; $c++) {
$col[$c] = $data[$c];}
$col1 = $col[1];
$sql = "SELECT Oldlocation FROM invbkup WHERE VIN = '$col1'"; ///works fine, gets old location for each record
$result = mysqli_query($con, $sql);
$row = mysqli_fetch_array($result);
$whl= $row['Oldlocation'];
$col8 = $col[73];
$query = "UPDATE allinv SET location = '$col8' WHERE $whl = 'DW' ";
echo $col1, $col8, $whl, "</br>";////can see all the info but get no changes in database.
$a= mysqli_query($con, $query);
}
fclose($handle);
}
Nevermind, it was a simple problem I fix by putting it in an if statement:
if($whl = 'DW'){
$query = "UPDATE allinv SET location = '$col8' WHERE VIN ='$col1'";
$a= mysqli_query($con, $query);}
sometimes you over complicate the problem yourself, take a break, drink some coffee and try not thinking so hard.
mysql_fetch_assoc in MeekroDB
Excuse me for being new to MeekroDB library, I wonder how to check for rows in the login process;
$query = DB::query("SELECT * FROM users WHERE username=%s",$username);
$numrows = DB::count($query);
// this needs the edit
if ($numrows!=0){
while($row = mysql_fetch_assoc($query)){
$dbusername = $row['username'];
$dbpassword = $row['password'];}
You need to bear in mind that your variable $query is now an array (if any records are found) so you do not need to do
while($row = mysql_fetch_assoc($query))
Instead try
foreach ($query as $user)
{
$dbusername = $user['username'];
$dbpassword = $user['password'];
}
Also, if you have more then one record with the same username and password then you may be doing something wrong with your user registration/maintenance process.
How to fetch single row/data from Mysql_fetch_array?
I am reading .csv file through PHP using fgetcsv(). Now I am able to fetch I mean read data from .csv file.
Then I need to check SKU column with my fetch result & accordingly have to perform either Insertion Or Updation.
Suppose SKU is already present there, then I have to update my row in table else I need to insert new record.
I have write following code...Plz check n tell me where I am doing mistake-:
<?php
$row = 1;
if (($handle = fopen("localhost/data/htdocs/magento/var/import/Price.csv", "r")) !== FALSE)
{
while (($data = fgetcsv($handle, 8000, ",")) !== FALSE)
{
$num = count($data);
echo "<p> $num fields in line $row: <br /></p>\n";
for ($c=0; $c < $num; $c++)
{
$temp = $data;
$string = implode(";", $temp);
}
$pieces = explode(";", $string);
$col1 = $pieces[0];
$col2 = $pieces[1];
$col3 = $pieces[2];
$col4 = $pieces[3];
$db_name = "magento";
$con = mysql_connect("localhost", "magento", "password");
If (!$con)
{
die('Could not connect: ' . mysql_error());
mysql_close($con);
}
$seldb = mysql_select_db($db_name, $con);
$query_fetch = "SELECT `sku` from `imoprt_prices`";
$result_fetch = mysql_query($query_fetch);
$num_rows = mysql_num_rows($result_fetch);
for($i = 0; $i < $num_rows; $i++)
{
$value = mysql_result($result_fetch, i, 'sku');
if(strcmp('".$value."', '".$col2."') == 0)
{
$flag = 1;
break;
}
else
{
$flag = 0;
break;
}
}
if($flag == 1)
{
$query_upadte = "(UPDATE imoprt_prices SET customer_id= '".$col1."', sku ='".$col2."', price= '".$col3."', website= '".$col4."'
)";
mysql_query($query_upadte);
$row++;
}
if($flag == 0)
{
mysql_query("INSERT INTO `imoprt_prices`(`customer_id`,`sku`,`price`,`website`) VALUES('".$col1."','".$col2."','".$col3."','".$col4."')");
$row++;
}
}
}
?>
If you have an actual UNIQUE index on your imoprt_prices table, you can use the ON DUPLICATE KEY UPDATE syntax and simplify your code a bit to something similar to; (note, can't test, so see as pseudo code)
$db_name = "magento";
$con = mysql_connect("localhost", "magento", "password") or die(mysql_error());
$seldb = mysql_select_db($db_name, $con);
if (($handle = fopen("localhost/data/htdocs/magento/var/import/Price.csv", "r")) !== FALSE)
{
while (($data = fgetcsv($handle, 8000, ",")) !== FALSE)
{
$col1 = $pieces[0];
$col2 = $pieces[1];
$col3 = $pieces[2];
$col4 = $pieces[3];
$query_upadte = "INSERT INTO imoprt_prices (customer_id, sku, price, website) ".
"VALUES('".$col1."','".$col2."','".$col3."','".$col4."') ".
"ON DUPLICATE KEY UPDATE customer_id='".$col1."', price='".$col3.
"',website='".$col4."'";
mysql_query($query_upadte);
}
}
You may also want to either mysql_real_escape_string() or use parameterized queries to make sure there's no ' in your inserted values though. That is always a danger with building sql strings like this.
Simple demo here.
In the following snippet: $query_upadte = "UPDATE imoprt_prices SET customer_id= '".$col1."', sku ='".$col2."', price= '".$col3."', website= '".$col4."'"; You're trying to update all the rows repeatedly, instead of just updating a single row. This is normally not allowed in MySQL. You need to specify a particular unique ID to be updated.