This page was generated from docs/examples/DataSet/Benchmarking.ipynb. Interactive online version: .

Dataset Benchmarking

This notebook is a behind-the-scenes benchmarking notebook, mainly for use by developers. The recommended way for users to interact with the dataset is via the Measurement object and its associated context manager. See the corresponding notebook for a comprehensive toturial on how to use those.

%matplotlib inline
from pathlib import Path

import numpy as np

import qcodes as qc
from qcodes.dataset import (
    ParamSpec,
    initialise_or_create_database_at,
    load_or_create_experiment,
    new_data_set,
)

qc.config.core.db_location

'~/experiments.db'

initialise_or_create_database_at(
    Path.cwd().parent / "example_output" / "benchmarking.db"
)

Setup

exp = load_or_create_experiment("benchmarking", sample_name="the sample is a lie")
exp

benchmarking#the sample is a lie#1@C:\Users\jensh\experiments.db
----------------------------------------------------------------

Now we can create a dataset. Note two things:

- if we don't specfiy a exp_id, but we have an experiment in the experiment container the dataset will go into that one.
- dataset can be created from the experiment object

dataSet = new_data_set("benchmark_data", exp_id=exp.exp_id)
exp

benchmarking#the sample is a lie#1@C:\Users\jensh\experiments.db
----------------------------------------------------------------
1-benchmark_data-1-None-0

In this benchmark we will assueme that we are doing a 2D loop and investigate the performance implications of writing to the dataset

x_shape = 100
y_shape = 100

Baseline: Generate data

%%time
rng = np.random.default_rng()
for x in range(x_shape):
    for y in range(y_shape):
        z = rng.random(1)

CPU times: total: 15.6 ms
Wall time: 10.9 ms

and store in memory

x_data = np.zeros((x_shape, y_shape))
y_data = np.zeros((x_shape, y_shape))
z_data = np.zeros((x_shape, y_shape))

%%time
rng = np.random.default_rng()
for x in range(x_shape):
    for y in range(y_shape):
        x_data[x, y] = x
        y_data[x, y] = y
        z_data[x, y] = rng.random()

CPU times: total: 0 ns
Wall time: 4.52 ms

Add to dataset inside double loop

double_dataset = new_data_set(
    "doubledata",
    exp_id=exp.exp_id,
    specs=[
        ParamSpec("x", "numeric"),
        ParamSpec("y", "numeric"),
        ParamSpec("z", "numeric"),
    ],
)
double_dataset.mark_started()

Note that this is so slow that we are only doing a 10th of the computation

%%time
rng = np.random.default_rng()
for x in range(x_shape // 10):
    for y in range(y_shape):
        double_dataset.add_results([{"x": x, "y": y, "z": rng.random()}])

CPU times: total: 922 ms
Wall time: 5.34 s

Add the data in outer loop and store as np array

single_dataset = new_data_set(
    "singledata",
    exp_id=exp.exp_id,
    specs=[ParamSpec("x", "array"), ParamSpec("y", "array"), ParamSpec("z", "array")],
)
single_dataset.mark_started()
x_data = np.zeros(y_shape)
y_data = np.zeros(y_shape)
z_data = np.zeros(y_shape)

%%time
rng = np.random.default_rng()
for x in range(x_shape):
    for y in range(y_shape):
        x_data[y] = x
        y_data[y] = y
        z_data[y] = rng.random(1)[0]
    single_dataset.add_results([{"x": x_data, "y": y_data, "z": z_data}])

CPU times: total: 156 ms
Wall time: 756 ms

Save once after loop

zero_dataset = new_data_set(
    "zerodata",
    exp_id=exp.exp_id,
    specs=[ParamSpec("x", "array"), ParamSpec("y", "array"), ParamSpec("z", "array")],
)
zero_dataset.mark_started()
x_data = np.zeros((x_shape, y_shape))
y_data = np.zeros((x_shape, y_shape))
z_data = np.zeros((x_shape, y_shape))

%%time
rng = np.random.default_rng()
for x in range(x_shape):
    for y in range(y_shape):
        x_data[x, y] = x
        y_data[x, y] = y
        z_data[x, y] = rng.random(1)[0]
zero_dataset.add_results([{"x": x_data, "y": y_data, "z": z_data}])

CPU times: total: 0 ns
Wall time: 17.3 ms

Array parameter

array1D_dataset = new_data_set(
    "array1Ddata",
    exp_id=exp.exp_id,
    specs=[ParamSpec("x", "array"), ParamSpec("y", "array"), ParamSpec("z", "array")],
)
array1D_dataset.mark_started()
y_setpoints = np.arange(y_shape)

%%timeit
rng = np.random.default_rng()
for x in range(x_shape):
    x_data[x, :] = x
    array1D_dataset.add_results(
        [{"x": x_data[x, :], "y": y_setpoints, "z": rng.random(y_shape)}]
    )

567 ms ± 52.1 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

x_data = np.zeros((x_shape, y_shape))
y_data = np.zeros((x_shape, y_shape))
z_data = np.zeros((x_shape, y_shape))
y_setpoints = np.arange(y_shape)

array0D_dataset = new_data_set(
    "array0Ddata",
    exp_id=exp.exp_id,
    specs=[ParamSpec("x", "array"), ParamSpec("y", "array"), ParamSpec("z", "array")],
)
array0D_dataset.mark_started()

%%timeit
rng = np.random.default_rng()
for x in range(x_shape):
    x_data[x, :] = x
    y_data[x, :] = y_setpoints
    z_data[x, :] = rng.random(y_shape)
array0D_dataset.add_results([{"x": x_data, "y": y_data, "z": z_data}])

5.66 ms ± 43.9 μs per loop (mean ± std. dev. of 7 runs, 100 loops each)

Insert many

rng = np.random.default_rng()
data = []
for i in range(100):
    for j in range(100):
        data.append({"x": i, "y": j, "z": rng.random()})

many_Data = new_data_set(
    "many_data",
    exp_id=exp.exp_id,
    specs=[
        ParamSpec("x", "numeric"),
        ParamSpec("y", "numeric"),
        ParamSpec("z", "numeric"),
    ],
)
many_Data.mark_started()

%%timeit
many_Data.add_results(data)

15.5 ms ± 946 μs per loop (mean ± std. dev. of 7 runs, 100 loops each)