Big SQL Select Query in Laravel - php

Question

I have table lids with under 430k entries.
How can I do SELECT query faster, optimized because now this query working under 5-7 minutes.
Heard something about Eloquent chunk in Laravel, but need clear PHP solution or with DB object.
Maybe with 'for' construction to process 100-1000 entries at a time, smthng like this.
Tried to do 'for' construction, but don't know how to do this optimized.
Be gentle with me pls :)
Want to know your opinion.
How can i upgrade it?
UPD: Did something like this
$count = $db->doQuery("SELECT count(*) FROM `lids`"); // db - my custom object with Database connection
$count = $count[0]['count(*)'];
$hundreds = $count / 100; // get 'for' counts
$start = 43; // id index starts from 43 in the table
$end = 142;
for ($index = 1; $index < $hundreds; $index++) {
$leads = [];
$leads = $db->doQuery("SELECT * FROM `lids` WHERE `id` BETWEEN " . $start . " AND " . $end . "");
// var_dump($leads);
// die();
if (empty($leads)) {
$start += 100;
$end += 100;
continue;
}
$uniques = [];
foreach ($leads as $lead) {
if (empty($lead['json_vars'])) continue;
$vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']); // hot fix for charset, from DB comes string in utf8mb4
$json = json_decode($vars, true);
if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
// check for unique values
if (!isset($uniques[$lead['id']]) || $uniques[$lead['id']]['phone'] != $lead['phone']) {
$uniques[$lead['id']] = $lead;
}
}
foreach ($uniques as $unique) {
$lid = Lid::create($unique);
}
$start += 100;
$end += 100;
// here i get residue of entries
if ($hundreds - $index < 1) {
$leads = $db->doQuery("SELECT * FROM `lids` WHERE `id` IN(" . ($count - $hundreds * 100) . ", " . $count . ") AND WHERE ");
foreach ($leads as $lead) {
$json = json_decode(preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']), true);
if (!str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
$lid = Lid::create($lead);
}
}
}
Upd2:
$db = new Database();
$counter = 0;
$scriptStart = date('d.m.Y H:i:s', strtotime('now'));
$lastRemote = $db->lastId('lids');
$lastInner = Lid::all(['id'])->last();
$lastInner = $lastInner->id;
$count = $lastRemote - $lastInner;
if ($count < 0) {
echo 'no new objects, canceled';
return false;
}
$start = $lastInner + 1;
$end = $lastRemote;
$fewEntries = false;
if ($count < 500) {
$fewEntries = true;
$index = $lastInner;
$hundreds = $lastRemote;
} else {
$index = 1;
$hundreds = $count / 100;
$end = $start + 99;
}
if ($fewEntries) {
$leads = $db->itemsBetween('lids', 'id', [$start, $end]);
$uniques = [];
foreach ($leads as $lead) {
if (empty($lead['json_vars'])) continue;
$vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
$json = json_decode($vars, true);
if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
if (
!isset($uniques[$lead['id']]) ||
$uniques[$lead['id']]['phone'] != $lead['phone'] &&
$uniques[$lead['id']]['ip'] != $lead['ip'] &&
$uniques[$lead['id']]['request_link'] != $lead['request_link']
) {
$uniques[$lead['id']] = $lead;
}
}
foreach ($uniques as $unique) {
$lid = Lid::create($unique);
$counter++;
}
} else {
for ($index; $index < $hundreds; $index++) {
$leads = [];
$leads = $db->itemsBetween('lids', 'id', [$start, $end]);
// var_dump($leads);
// die();
$uniques = [];
foreach ($leads as $lead) {
if (empty($lead['json_vars'])) continue;
$vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
$json = json_decode($vars, true);
if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
if (
!isset($uniques[$lead['id']]) ||
$uniques[$lead['id']]['phone'] != $lead['phone'] &&
$uniques[$lead['id']]['ip'] != $lead['ip'] &&
$uniques[$lead['id']]['request_link'] != $lead['request_link']
) {
$uniques[$lead['id']] = $lead;
}
}
foreach ($uniques as $unique) {
$lid = Lid::create($unique);
$counter++;
}
$start += 100;
$end += 100;
}
}
$scriptEnd = date('d.m.Y H:i:s', strtotime('now'));
echo 'added in table: ' . $counter . PHP_EOL;
echo 'started at: ' . $scriptStart . PHP_EOL;
echo 'ended at: ' . $scriptEnd . PHP_EOL;

Answer 1

Resolved my problem with optimization of my trash code XD
$db = new Database();
$counter = 0;
$scriptStart = date('d.m.Y H:i:s', strtotime('now'));
$lastRemote = $db->lastId('lids');
$lastInner = Lid::all(['id'])->last();
$lastInner = $lastInner->id;
$count = $lastRemote - $lastInner;
if ($count < 0) {
echo 'no new objects, canceled';
return false;
}
$start = $lastInner + 1;
$end = $lastRemote;
$fewEntries = false;
if ($count < 500) {
$fewEntries = true;
$index = $lastInner;
$hundreds = $lastRemote;
} else {
$index = 1;
$hundreds = $count / 100;
$end = $start + 99;
}
if ($fewEntries) {
$leads = $db->itemsBetween('lids', 'id', [$start, $end]);
$uniques = [];
foreach ($leads as $lead) {
if (empty($lead['json_vars'])) continue;
$vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
$json = json_decode($vars, true);
if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
if (
!isset($uniques[$lead['id']]) ||
$uniques[$lead['id']]['phone'] != $lead['phone'] &&
$uniques[$lead['id']]['ip'] != $lead['ip'] &&
$uniques[$lead['id']]['request_link'] != $lead['request_link']
) {
$uniques[$lead['id']] = $lead;
}
}
foreach ($uniques as $unique) {
$lid = Lid::create($unique);
$counter++;
}
} else {
for ($index; $index < $hundreds; $index++) {
$leads = [];
$leads = $db->itemsBetween('lids', 'id', [$start, $end]);
// var_dump($leads);
// die();
$uniques = [];
foreach ($leads as $lead) {
if (empty($lead['json_vars'])) continue;
$vars = preg_replace('/[\x00-\x1F\x80-\xFF]/', '', $lead['json_vars']);
$json = json_decode($vars, true);
if (!isset($json['ser']['HTTP_COOKIE']) || !str_contains($json['ser']['HTTP_COOKIE'], '_fbc')) continue;
$lead['json_vars'] = $json['ser']['HTTP_COOKIE'];
if (
!isset($uniques[$lead['id']]) ||
$uniques[$lead['id']]['phone'] != $lead['phone'] &&
$uniques[$lead['id']]['ip'] != $lead['ip'] &&
$uniques[$lead['id']]['request_link'] != $lead['request_link']
) {
$uniques[$lead['id']] = $lead;
}
}
foreach ($uniques as $unique) {
$lid = Lid::create($unique);
$counter++;
}
$start += 100;
$end += 100;
}
}
$scriptEnd = date('d.m.Y H:i:s', strtotime('now'));
echo 'added in table: ' . $counter . PHP_EOL;
echo 'started at: ' . $scriptStart . PHP_EOL;
echo 'ended at: ' . $scriptEnd . PHP_EOL;