cma.test

doctest all (listed) files of the cma package.

Details: accepts verbose and all other keyword arguments that doctest.testfile would accept, while negative verbose values are passed as 0.

Undocumented

bytes (in Python 3) also fit the bill

test the cma package.

The first argument can be '-h' or '--help' or 'list' to list all files to be tested. Otherwise, arguments can be file(name)s to be tested, where names are interpreted relative to the package root and a leading 'cma' + path separator is ignored.

By default all files are tested.

various doc tests.

This function describes test cases and might in future become helpful as an experimental tutorial as well. The main testing feature at the moment is by doctest with cma.test.main() in a Python shell or by python -m cma.test in a system shell.

A simple first overall test:

>>> import cma
>>> res = cma.fmin(cma.ff.elli, 3*[1], 1,
...                {'CMA_diagonal':2, 'seed':1, 'verbose':-9})
>>> assert res[1] < 1e-6
>>> assert res[2] < 2000

Testing args argument:

>>> def maxcorr(m):
...     val = 0
...     for i in range(len(m)):
...         for j in range(i + 1, len(m)):
...             val = max((val, abs(m[i, j])))
...     return val
>>> x, es = cma.fmin2(cma.ff.elli, [1, 0, 0, 0], 0.5, {'verbose':-9}, args=[True])  # rotated
>>> assert maxcorr(es.sm.correlation_matrix) > 0.85, es.sm.correlation_matrix
>>> es = cma.CMAEvolutionStrategy([1, 0, 0, 0], 0.5,
...                               {'verbose':-9}).optimize(cma.ff.elli, args=[1])
>>> assert maxcorr(es.sm.correlation_matrix) > 0.85, es.sm.correlation_matrix

Testing output file consistency with diagonal option:

>>> import cma
>>> for val in (0, True, 2, 3):
...     _ = cma.fmin(cma.ff.sphere, 3 * [1], 1,
...                  {'verb_disp':0, 'CMA_diagonal':val, 'maxiter':5})
...     _ = cma.CMADataLogger().load()

Test on the Rosenbrock function with 3 restarts. The first trial only finds the local optimum, which happens in about 20% of the cases.

>>> import cma
>>> res = cma.fmin(cma.ff.rosen, 4 * [-1], 0.01,
...                options={'ftarget':1e-6,
...                     'verb_time':0, 'verb_disp':500,
...                     'seed':3},
...                restarts=3)
...                # doctest: +ELLIPSIS
(4_w,8)-aCMA-ES (mu_w=2.6,w_1=52%) in dimension 4 (seed=3,...)
Iterat #Fevals ...
>>> assert res[1] <= 1e-6

Notice the different termination conditions. Termination on the target function value ftarget prevents further restarts.

Test of scaling_of_variables option

>>> import cma
>>> opts = cma.CMAOptions()
>>> opts['seed'] = 4567
>>> opts['verb_disp'] = 0
>>> opts['CMA_const_trace'] = True
>>> # rescaling of third variable: for searching in  roughly
>>> #   x0 plus/minus 1e3*sigma0 (instead of plus/minus sigma0)
>>> opts['scaling_of_variables'] = [1, 1, 1e3, 1]
>>> res = cma.fmin(cma.ff.rosen, 4 * [0.1], 0.1, opts)
>>> assert res[1] < 1e-9
>>> es = res[-2]
>>> es.result_pretty()  # doctest: +ELLIPSIS
termination on tolfun=1e-11
final/bestever f-value = ...

The printed std deviations reflect the actual value in the parameters of the function (not the one in the internal representation which can be different).

Test of CMA_stds scaling option.

>>> import cma
>>> opts = cma.CMAOptions()
>>> s = 5 * [1]
>>> s[0] = 1e3
>>> opts.set('CMA_stds', s)  #doctest: +ELLIPSIS
{'...
>>> opts.set('verb_disp', 0)  #doctest: +ELLIPSIS
{'...
>>> res = cma.fmin(cma.ff.cigar, 5 * [0.1], 0.1, opts)
>>> assert res[1] < 1800

Testing combination of fixed_variables and CMA_stds options.

>>> import cma
>>> options = {
...     'fixed_variables':{1:2.345},
...     'CMA_stds': 4 * [1],
...     'minstd': 3 * [1]}
>>> es = cma.CMAEvolutionStrategy(4 * [1], 1, options) #doctest: +ELLIPSIS
(3_w,7)-aCMA-ES (mu_w=2.3,w_1=58%) in dimension 3 (seed=...

Test of elitism:

>>> import cma
>>> res = cma.fmin(cma.ff.rastrigin, 10 * [0.1], 2,
...       {'CMA_elitist':'initial', 'ftarget':1e-3, 'verbose':-9})
>>> assert 'ftarget' in res[7]

Test CMA_on option and similar:

>>> import cma
>>> res = cma.fmin(cma.ff.sphere, 4 * [1], 2,
...      {'CMA_on':False, 'ftarget':1e-8, 'verbose':-9})
>>> assert 'ftarget' in res[7] and res[2] < 1e3
>>> res = cma.fmin(cma.ff.sphere, 3 * [1], 2,
...      {'CMA_rankone':0, 'CMA_rankmu':0, 'ftarget':1e-8,
...       'verbose':-9})
>>> assert 'ftarget' in res[7] and res[2] < 1e3
>>> res = cma.fmin(cma.ff.sphere, 2 * [1], 2,
...      {'CMA_rankone':0, 'ftarget':1e-8, 'verbose':-9})
>>> assert 'ftarget' in res[7] and res[2] < 1e3
>>> res = cma.fmin(cma.ff.sphere, 2 * [1], 2,
...      {'CMA_rankmu':0, 'ftarget':1e-8, 'verbose':-9})
>>> assert 'ftarget' in res[7] and res[2] < 1e3

Check rotational invariance:

>>> import cma
>>> felli = cma.s.ft.Shifted(cma.ff.elli)
>>> frot = cma.s.ft.Rotated(felli)
>>> res_elli = cma.CMAEvolutionStrategy(3 * [1], 1,
...           {'ftarget': 1e-8}).optimize(felli).result
...                  #doctest: +ELLIPSIS
(3_w,7)-...
>>> res_rot = cma.CMAEvolutionStrategy(3 * [1], 1,
...         {'ftarget': 1e-8}).optimize(frot).result
...                  #doctest: +ELLIPSIS
(3_w,7)-...
>>> assert res_rot[3] < 2 * res_elli[3]

Both condition alleviation transformations are applied during this test, first in iteration 62ish, second in iteration 257ish:

>>> import warnings
>>> import cma
>>> ftabletrot = cma.fitness_transformations.Rotated(cma.ff.tablet, seed=10)
>>> es = cma.CMAEvolutionStrategy(4 * [1], 1, {
...                                   'tolconditioncov':False,
...                                   'seed': 8,
...                                   'CMA_mirrors': 0,
...                                   'CMA_diagonal_decoding': 0,
...                                   'ftarget': 1e-8,
...                                })  # doctest:+ELLIPSIS
(4_w...
>>> while not es.stop() and es.countiter < 90:
...     X = es.ask()
...     es.tell(X, [cma.ff.elli(x, cond=1e22) for x in X])  # doctest:+ELLIPSIS
NOTE ...iteration=...
>>> with warnings.catch_warnings(record=True) as warns:
...     while not es.stop():
...         X = es.ask()
...         es.tell(X, [ftabletrot(x) for x in X])  # doctest:+ELLIPSIS
>>> assert not warns or isinstance(warns[0].message, UserWarning)
>>> assert es.countiter <= 355 and 'ftarget' in es.stop(), (
...             "transformation bug in alleviate_condition?",
...             es.countiter, es.stop())

Integer handling:

>>> import warnings
>>> idx = [0, 1, -1]
>>> f = cma.s.ft.IntegerMixedFunction2(cma.ff.elli, idx)
>>> for more_opts in [{}, {'AdaptSigma': cma.sigma_adaptation.CMAAdaptSigmaTPA}]:
...     opts = dict(ftarget=1e-9, seed=5, verbose=-9, integer_variables=idx)
...     opts.update(more_opts)
...     es = cma.CMAEvolutionStrategy(4 * [5], 10, opts).optimize(f)
...     assert 'ftarget' in es.stop() and es.result[3] < 1800
>>> # mixing integer and fixed variables
>>> es = cma.CMA(5 * [1], 1, {'verbose':-9, 'integer_variables':[1,2,4],
...                           'fixed_variables':{1:0}})
>>> assert es.opts['integer_variables'] == [1, 3]
>>> # TODO: do more testing here or in the class

Parallel objective:

>>> def parallel_sphere(X): return [cma.ff.sphere(x) for x in X]
>>> x, es = cma.fmin2(cma.ff.sphere, 3 * [0], 0.1, {
...     'verbose': -9, 'eval_final_mean': True, 'CMA_elitist': 'initial'},
...                   parallel_objective=parallel_sphere)
>>> assert es.result[1] < 1e-9
>>> x, es = cma.fmin2(None, 3 * [0], 0.1, {
...     'verbose': -9, 'eval_final_mean': True, 'CMA_elitist': 'initial'},
...                   parallel_objective=parallel_sphere)
>>> assert es.result[1] < 1e-9

Some sort of interactive control via an options file:

>>> es = cma.CMAEvolutionStrategy(4 * [2], 1, dict(
...                      signals_filename='cma_signals.in',
...                      verbose=-9))
>>> s = es.stop()
>>> es = es.optimize(cma.ff.sphere)

Test of huge lambda:

>>> es = cma.CMAEvolutionStrategy(3 * [0.91], 1, {
...     'verbose': -9,
...     'popsize': 200,
...     'ftarget': 1e-8 })
>>> es = es.optimize(cma.ff.tablet)
>>> if es.result.evaluations > 5000: print(es.result.evalutions, es.result)

For VD- and VkD-CMA, see cma.restricted_gaussian_sampler.

>>> import sys
>>> import cma
>>> assert cma.interfaces.EvalParallel2 is not None
>>> try:
...     with warnings.catch_warnings(record=True) as warn:
...         with cma.optimization_tools.EvalParallel2(cma.ff.elli) as eval_all:
...             res = eval_all([[1,2], [3,4]])
... except:
...     assert sys.version[0] == '2'