Doctrine batch inserting uses 2GB of Ram - php

Question

I am trying to insert approximately 200k of records with Symfony2 and Doctrine via Doctrine Fixtures Bundle. I am using flush and clear but at the end script uses 1.8 GB of RAM.
This is the class that loads SmartMeter entities into database:
<?php
namespace HTEC\SmartMeteringAPIBundle\DataFixtures\ORM;
use Doctrine\Common\DataFixtures\AbstractFixture;
use Doctrine\Common\DataFixtures\FixtureInterface;
use Doctrine\Common\Persistence\ObjectManager;
use Doctrine\Common\DataFixtures\OrderedFixtureInterface;
use HTEC\SmartMeteringAPIBundle\Entity\SmartMeter;
use Symfony\Component\DependencyInjection\ContainerAwareInterface;
use Symfony\Component\DependencyInjection\ContainerInterface;
class LoadSmartMeterData extends AbstractFixture implements FixtureInterface, ContainerAwareInterface, OrderedFixtureInterface
{
static $NUMBER_OF_SMART_METERS = 0;
static $MAX_NUM_OF_SM_PER_CONC = 500;
/**
* #var ContainerInterface
*/
private $container;
public function getOrder()
{
return 10; // the order in which fixtures will be loaded
}
/**
* #inheritDoc
*/
public function setContainer(ContainerInterface $container = null)
{
$this->container = $container;
}
/**
* {#inheritDoc}
*/
public function load(ObjectManager $manager)
{
$numberOfUsers = LoadUserData::$NUMBER_OF_USERS;
$numberOfConcentrators = LoadConcentratorData::$NUMBER_OF_CONCENTRATORS;
$numberOfTariffs = LoadTariffData::$NUMBER_OF_TARIFFS;
$numberOfSmartMetersPerConcentrator = 0;
$smartMeter = null;
$concentrator = null;
$user = null;
$tariff = null;
$feeders = array();
$numberOfFeeders = 0;
$feedersRandomKey = array();
$isIpAddressDynamic = false;
$numberOfTransformerFeeders = 0;
$smartMeterType = 0;
$smartMeterProtocol = 0;
$modemType = 0;
$numberOfSmartMeters = 0;
echo "\n\nCreating Smart Meters. This could take a couple of minutes and could take approximately 2 GB of RAM \n\n...";
$startTime = time();
$smartMeterCount = 0;
$lastChunkNumber = 0;
$currentChunkNumber = 0;
$lastSmartMeterNumber = 0;
$smartMetersClearedCurrently = 0;
$concentratorSmartMeters = null;
for($i = 0; $i < LoadConcentratorData::$NUMBER_OF_CONCENTRATORS; $i++)
{
$concentrator = $manager->getRepository('SMAPIBundle:Concentrator')->find($i+1);
$numberOfSmartMetersPerConcentrator = rand(1, self::$MAX_NUM_OF_SM_PER_CONC);
$numberOfSmartMeters += $numberOfSmartMetersPerConcentrator;
for($c = 0; $c < $numberOfSmartMetersPerConcentrator; $c++)
{
$smartMeter = new SmartMeter();
$smartMeter->setSerialNumber(++$smartMeterCount);
$smartMeter->setConcentrator($concentrator);
$smartMeterType = rand(1,3);
switch($smartMeterType)
{
case 1:
$smartMeter->setType('DIRECT');
break;
case 2:
$smartMeter->setType('HALF');
break;
case 3:
$smartMeter->setType('INDIRECT');
break;
}
$user = $manager->getRepository('SMAPIBundle:User')->find(rand(1, $numberOfUsers));
$smartMeter->setCreatedBy($user);
$numberOfTransformerFeeders = $concentrator->getTransformerFeeders()->count();
$feeders = $concentrator->getTransformerFeeders()->toArray();
if($numberOfTransformerFeeders > 0)
{
$feedersRandomKey = array_rand($feeders, 1);
if(isset($feeders[$feedersRandomKey]))
{
$smartMeter->setFeeder($feeders[$feedersRandomKey]);
}
}
$smartMeter->setStatus(rand(0,2));
$tariff = $manager->getRepository('SMAPIBundle:Tariff')->find(rand(1, $numberOfTariffs));
$smartMeter->setTariff($tariff);
$smartMeterProtocol = rand(1,3);
switch($smartMeterProtocol)
{
case 1:
$smartMeter->setProtocol('DLMS');
break;
case 2:
$smartMeter->setProtocol('EURIDIS');
break;
case 3:
$smartMeter->setProtocol('IEC');
break;
}
$smartMeter->setModemSerialNumber(rand(1, 9000000000));
$smartMeter->setManufacture('MAN: ' . ($i + $c));
$modemType = rand(1,2);
if($modemType === 1)
{
$smartMeter->setModemType('PLC');
$smartMeter->setModemIndex(rand(1, 512));
}
else
{
$smartMeter->setModemType('GPRS');
$isIpAddressDynamic = rand(0,1);
if($isIpAddressDynamic === 1)
{
$smartMeter->setModemIpAddress(rand(1000000, 2000000));
}
else
{
$smartMeter->setModemPhoneNumber($this->getRandomIpAddressV4());
}
}
if(rand(0,1) === 1)
{
$smartMeter->setModemRepeaterNumber(rand(10000000, 90000000));
}
$manager->persist($smartMeter);
} // end of FOR numberOfSmartMetersPerConcentrator
// flush smart meters
$manager->flush();
$manager->clear();
$currentChunkNumber = ceil($smartMeterCount / 5000);
if($smartMeterCount > 5000 && $currentChunkNumber > $lastChunkNumber)
{
$lastChunkNumber = $currentChunkNumber;
if($lastSmartMeterNumber > 0)
{
$smartMetersClearedCurrently = $smartMeterCount - $lastSmartMeterNumber;
}
else
{
$smartMetersClearedCurrently = $smartMeterCount;
}
echo "\n\nFlushing and clearing " . number_format($smartMetersClearedCurrently, 0, ',', '.') . " Smart Meters.\nTotal memory used after flush and clear: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";
$lastSmartMeterNumber = $smartMeterCount;
}
}// end for NUMBER_OF_CONCENTRATORS
$manager->flush();
$manager->clear();
unset($concentrator);
unset($smartMeter);
unset($tariff);
echo "\n\n-------------------------------------------\n\n";
echo "\n\nTotal memory used after final flush and clear of Smart Meters: " . number_format(((memory_get_usage(true) / 1024) / 1024), 2, ',', '.') . " Megabytes\n\n...";
echo "\n\nCreating Smart Meters complete. Created " . number_format($numberOfSmartMeters, 0, ',', '.') . " Smart Meters.\n\n";
$durationSeconds = (time() - $startTime);
$durationMinutes = $durationSeconds / 60;
$secondsRemainder = $durationSeconds % 60;
echo "\n\nTotal duration time: " . ceil($durationMinutes) . " minutes and " . $secondsRemainder . " seconds.\n\n\n";
self::$NUMBER_OF_SMART_METERS = $numberOfSmartMeters;
}
public function getRandomIpAddressV4()
{
return rand(1, 255) . '.' . rand(0, 255) . '.' . rand(0, 255) . '.' . rand(0, 255);
}
}
On approximately 5000 records I am calling flush and clear but it seams that no memory is released.
Are there recommendations about how to prevent Doctrine from using too much RAM during batch tasks?

Answer 1

If it is a command, try running it with the option --no-debug. Or, you can disable the logger by calling $manager->getConnection()->getConfiguration()->setSQLLogger(null); at the beginning of your loadfunction.
Either way disabling the logger saves quite a lot of memory during Doctrine batch tasks.