More information on what to test

There are currently independent efforts underway to improve the testing for the IMAP and GD extension. Therefore these two extensions should not be focused on during a testfest in May 08.

You can tell a certain amount from line coverage - but it doesn't really show you exactly what tests there are already for specific functions. So how can you tell?

I've put the source code for three scripts below - they are a little unfinished, for example they all produce warning messages (which I think can safely be ignored).

The first two scripts will tell you about functions that are not tested at all and which tests call a given function. The third script does something similar for class methods. This isn't quite as easy as looking for functions, but there is still some useful information.

This script runs through all of the phpt tests in the source tree and then compares the function calls in the tests with a complete list of PHP's internal functions. At the end it will tell you if there is a function in PHP that is not mentioned in any test case (I get 415). Thanks to Damien Seguy for this.

<?php
 
// define here your PHP source directory
define('PHP_SRC','/home/zoe/BUILDS/php53_dev');
 
//Get a list of all of the test (phpt) files in the source tree.
$files = array();
get_phpt_files(PHP_SRC, &$file_count, &$files);
 
//Find all the functions that are called in the test case code
$tested = array();
foreach($files as $file) {
        $tested = array_merge(extract_tests($file), $tested);
}
$tested = array_unique($tested);
//print_r($tested);
 
//Get a list of all of the internal PHP functions
$internal = get_defined_functions();
$internal = $internal['internal'];
 
//Compare the list of all function with those that are called from a phpt test case.
$non_tested = array_diff($internal, $tested);
print_r($non_tested);
print count($non_tested). " untested functions\n";
 
 
function  extract_tests($file) {
        $code = file_get_contents($file);
 
        if (!preg_match('/--FILE--\s*(.*)\s*--(EXPECTF|EXPECTREGEX|EXPECT)?--/is', $code, $r)) {
//              print "Unable to get code in ".$file."\n";
                return array();
        }
 
        $tokens = token_get_all($r[1]);
        $functions = array_filter($tokens, 'filter_functions');
        $functions = array_map( 'map_token_value',$functions);
        $functions = array_unique($functions);
 
        return $functions;
}
 
function filter_functions($x) {
        return $x[0] == 307;
}
 
function map_token_value($x) {
        return $x[1];
}
function get_phpt_files($dir, &$phpt_file_count, &$all_phpt)
{
        $thisdir = dir($dir.'/'); //include the trailing slash
        while(($file = $thisdir->read()) !== false) {
                if ($file != '.' && $file != '..') {
                        $path = $thisdir->path.$file;
                                if(is_dir($path) == true) {
                                        get_phpt_files($path , $phpt_file_count , $all_phpt);
                                } else {
                                        if (preg_match("/\w+\.phpt$/", $file)) {
                                                $all_phpt[$phpt_file_count] = $path;
                                                $phpt_file_count++;
                                        }
                                }
                }
        }
return;
}
 
 
?>

A slight modification to Damien's script. In this case the script will print out what test cases each function is mentioned in. This doesn't mean that the function is properly tested in each one, just that it's used. For example, if you look at the output for 'cos' you will see three tests which are clearly meant to be testing 'cos'. Look at the output for 'var_dump' and you'll see that it's used in 100's of tests - but is it really tested in any of them?

<?php
// define here your PHP source directory
define('PHP_SRC','/home/zoe/BUILDS/php53_dev');
 
//Get a list of all of the test (phpt) files in the source tree.
$files = array();
get_phpt_files(PHP_SRC, &$file_count, &$files);
 
 
$tests_by_function=array();
 
//Get a list of all of the internal PHP functions
$internal = get_defined_functions();
$internal = $internal['internal'];
foreach($internal as $function) {
        $tests_by_function[$function] = array();
}
 
//Find all the functions that are called in the test case code
$tested = array();
$temp_array = array();
 
foreach($files as $file) {
        $temp_array = extract_tests($file);
        $tested = array_merge($temp_array, $tested);
        foreach($temp_array as $function) {
                if(array_key_exists($function, $tests_by_function)) {
                        array_push($tests_by_function[$function],$file);
                }
        }
}
$tested = array_unique($tested);
print_r($tests_by_function);
 
 
function  extract_tests($file) {
        $code = file_get_contents($file);
 
        if (!preg_match('/--FILE--\s*(.*)\s*--(EXPECTF|EXPECTREGEX|EXPECT)?--/is', $code, $r)) {
//              print "Unable to get code in ".$file."\n";
                return array();
        }
 
        $tokens = token_get_all($r[1]);
        $functions = array_filter($tokens, 'filter_functions');
        $functions = array_map( 'map_token_value',$functions);
        $functions = array_unique($functions);
 
        return $functions;
}
 
function filter_functions($x) {
        return $x[0] == 307;
}
 
function map_token_value($x) {
        return $x[1];
}
function get_phpt_files($dir, &$phpt_file_count, &$all_phpt)
{
        $thisdir = dir($dir.'/'); //include the trailing slash
        while(($file = $thisdir->read()) !== false) {
                if ($file != '.' && $file != '..') {
                        $path = $thisdir->path.$file;
                                if(is_dir($path) == true) {
                                        get_phpt_files($path , $phpt_file_count , $all_phpt);
                                } else {
                                        if (preg_match("/\w+\.phpt$/", $file)) {
                                                $all_phpt[$phpt_file_count] = $path;
                                                $phpt_file_count++;
                                        }
                                }
                }
        }
return;
}
 
 
?>

This script lists all the classes in PHP and then finds their methods using a ReflectionClass. Using essentially the same logic as the other two scripts teh code searches for tests that refer to these methods. The script is flawed in that many classes have methods of the same name (for example __toString occurs in many classes), so looking for all tests that refer to __toString will give a list of mainly irrelevant tests. Because of this I've excluded any method beginning “__” from the search, even so, the results should be treated with care.

<?php
// define here your PHP source directory
define('PHP_SRC','/home/zoe/BUILDS/php53_dev');
 
//Get a list of all of the test (phpt) files in the source tree.
$files = array();
get_phpt_files(PHP_SRC, &$file_count, &$files);
 
 
$tests_by_method=array();
$class_by_method=array();
 
//Get a list of all of the PHP classes
$classes = get_declared_classes();
 
$methodArray = array();
$methodNameArray = array();
$method_names = array();
 
//Get the method names from each class
foreach($classes as $class) {
        $classInfo = new ReflectionClass($class);
        $methodArray = $classInfo->getMethods();
        $method_names = get_method_names($methodArray);
        $class_by_method[$class] = array();
        array_push($class_by_method[$class], $method_names);
        $methodNameArray = array_merge($method_names, $methodNameArray);
        foreach($method_names as $method) {
                $tests_by_method[$method] = array();
        }
}
 
//Find all the functions that are called in the test case code
$tested = array();
$temp_array = array();
 
foreach($files as $file) {
        $temp_array = extract_tests($file);
        $tested = array_merge($temp_array, $tested);
        foreach($temp_array as $function) {
                if(array_key_exists($function, $tests_by_method)) {
                        array_push($tests_by_method[$function],$file);
                }
        }
}
//Print out the tests in which methods are used
//print_r ($class_by_method);
foreach($classes as $class) {
        echo "\n\nCLASS NAME : ".$class."\n";
        foreach($class_by_method[$class] as $methods) {
                foreach($methods as $method) {
                        if(!preg_match('/^__.*/', $method)) { //Exclude methods like __toString, there are too many of them
                                echo "Method name: ".$method."\n";
                                foreach($tests_by_method[$method] as $test) {
                                        echo "Test: ".$test."\n";
                                }
                        }
                }
      }
}
 
 
function  extract_tests($file) {
        $code = file_get_contents($file);
 
        if (!preg_match('/--FILE--\s*(.*)\s*--(EXPECTF|EXPECTREGEX|EXPECT)?--/is', $code, $r)) {
//              print "Unable to get code in ".$file."\n";
                return array();
        }
 
        $tokens = token_get_all($r[1]);
        $functions = array_filter($tokens, 'filter_functions');
        $functions = array_map( 'map_token_value',$functions);
        $functions = array_unique($functions);
        return $functions;
}
 
function filter_functions($x) {
        return $x[0] == 307;
}
 
function map_token_value($x) {
        return $x[1];
}
function get_phpt_files($dir, &$phpt_file_count, &$all_phpt)
{
        $thisdir = dir($dir.'/'); //include the trailing slash
        while(($file = $thisdir->read()) !== false) {
                if ($file != '.' && $file != '..') {
                        $path = $thisdir->path.$file;
                                if(is_dir($path) == true) {
                                        get_phpt_files($path , $phpt_file_count , $all_phpt);
                                } else {
                                        if (preg_match("/\w+\.phpt$/", $file)) {
                                                $all_phpt[$phpt_file_count] = $path;
                                                $phpt_file_count++;
                                        }
                                }
                }
        }
return;
}
function get_method_names($methodArray) {
        $names = array();
        foreach ($methodArray as $method) {
                array_push($names,$method->name);
        }
        return $names;
}
 
?>