Saturday, 23 June 2012

PHP memory consumption with Arrays and Objects (update: generators)

Lessons learned:
  • objects need more memory than arrays (+ 2-10 percent)
  • comparing 32bit to 64bit systems, memory consumption increases by 100-230 percent
  • if array values are numeric, don't save them as strings!
  • saving 1M integers takes 200M of memory with PHP on a 64bit plattform
    (increase by factor 25)
  • using SplFixedArray can reduce memory usage by 20-130 percent
  • avoid big Arrays and Objects in PHP whenever possible
    (don't use file('big_file') or explode("\n", file_get_contents('big_file')), etc.)
  • use streams whenever possible (fopen, fsockopen, etc.)
  • use generators when available with PHP 5.5 (RFC)
If you need to save memory temporarily in your script, you can stringify your array (increasing cpu usage and runtime):

$a = array();
for ($i=0; $i<1000000; $i++) {
$a[] = $i;
}
$a = implode(',', $a); // or $a = json_encode($a);

echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 8.0 M (64bit), 7.5 M (32bit)
Here is a small script to show how much memory PHP needs to handle bit Arrays:
(Tests made with PHP 5.3.10 and PHP 5.4.0)

ini_set('memory_limit', '1024M');

$a = '';
for ($i=0; $i<1000000; $i++) {
$a .= '1';
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 1.75 M (64bit), 1.25 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = true;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 199.75 M (64bit), 80.75 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = true;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 207.50 M (64bit), 88.50 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = true;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 84.75 M (64bit), 34.75 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = $i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 199.75 M (64bit), 80.75 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = $i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 207.50 M (64bit), 88.50 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = $i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 84.75 M (64bit), 34,75 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = (string)$i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 322.00 M (64bit), 164.75 M (32bit)

$a = new stdclass();
for ($i=0; $i<1000000; $i++) {
$a->$i = (string)$i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 329.50 M (64bit), 172.50 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = (string)$i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 207.00 M (64bit), 118.75 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[(string)$i] = (string)$i;
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 321.75 M (64bit), 164.75 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = "";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 253.25 M (64bit), 96.00 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = "";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 261.00 M (64bit), 103.75 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = "";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 138.50 M (64bit), 50.25 M (32bit)



$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = "hello";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 253.25 M (64bit), 96.00 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = "hello";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 261.00 M (64bit), 103.75 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = "hello";
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 138.50 M (64bit), 50.25 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = array();
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 421.25 M (64bit), 126.50 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = array();
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 428.75 M (64bit), 134.25 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = array();
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 306.25 M (64bit), 50.75 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = array("");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 665.25 M (64bit), 256.25 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = array("");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 673.00 M (64bit), 264.00 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = array("");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 550.50 M (64bit), 210.50 M (32bit)


$a = array();
for ($i=0; $i<1000000; $i++) {
$a[$i] = array("hello");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 665.25 M (64bit), 256.25 M (32bit)

$a = new stdclass;
for ($i=0; $i<1000000; $i++) {
$a->$i = array("hello");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 673.00 M (64bit), 264.00 M (32bit)

$a = new SplFixedArray(1000000);
for ($i=0; $i<1000000; $i++) {
$a[$i] = array("hello");
}
echo number_format(memory_get_usage(true)/1048576, 2)."\n";
// 550.50 M (64bit), 210.50 M (32bit)

More details and technical explanations about PHP's memory usage can be found on nikic's blog.

Shortly: a string or an integer in PHP are not mapped to a char-array or an integer in C. A more complex structure is used to get from dynamic typing in PHP to static typing in C:


struct _zval_struct {
zvalue_value value; // value object
zend_uint refcount__gc; // number of references to variable
zend_uchar type; // type of variable
zend_uchar is_ref__gc; // reference? (&$var)
};
typedef union _zvalue_value {
long lval; // integer and boolean
double dval; // float (double)
struct {
char *val; // string (with zero bytes)
int len; // length of the string
} str;
HashTable *ht; // array (hash table)
zend_object_value obj; // object
} zvalue_value;

No comments:

Post a Comment