I Love Symposia! (Posts about open-source)

Note: · alone

int callback(double *buffer, intptr_t filter_size, 
             double *return_value, void *user_data)

>>> import numpy as np
>>> footprint = np.array([[0, 1, 0],
...                       [1, 1, 1],
...                       [0, 1, 0]], dtype=bool)
>>> from scipy import ndimage as ndi
>>>
>>> %timeit ndi.grey_erosion(image, footprint=fp)
1 loop, best of 3: 160 ms per loop

import numba
from numba import cfunc, carray
from numba.types import intc, CPointer, float64, intp, voidptr
from scipy import LowLevelCallable

def jit_filter_function(filter_function):
    jitted_function = numba.jit(filter_function, nopython=True)
    @cfunc(intc(CPointer(float64), intp, CPointer(float64), voidptr))
    def wrapped(values_ptr, len_values, result, data):
        values = carray(values_ptr, (len_values,), dtype=float64)
        result[0] = jitted_function(values)
        return 1
    return LowLevelCallable(wrapped.ctypes)

@jit_filter_function
def fmin(values):
    result = np.inf
    for v in values:
        if v < result:
            result = v
    return result

>>> %timeit ndi.generic_filter(image, fmin, footprint=fp)
10 loops, best of 3: 92.9 ms per loop

from llc import jit_filter_function

>>> import numpy as np
>>> image = np.random.random((2048, 2048))

>>> footprint = np.array([[0, 1, 0],
...                       [1, 1, 1],
...                       [0, 1, 0]], dtype=bool)

>>> from scipy import ndimage as ndi
>>> %timeit ndi.grey_erosion(image, footprint=footprint)
10 loops, best of 3: 118 ms per loop
>>> %timeit ndi.generic_filter(image, np.min, footprint=footprint)
1 loop, best of 3: 27 s per loop

>>> from numba import cfunc, carray
>>> from numba.types import intc, intp, float64, voidptr
>>> from numba.types import CPointer
>>> 
>>> 
>>> @cfunc(intc(CPointer(float64), intp,
...             CPointer(float64), voidptr))
... def nbmin(values_ptr, len_values, result, data):
...     values = carray(values_ptr, (len_values,), dtype=float64)
...     result[0] = np.inf
...     for v in values:
...         if v < result[0]:
...             result[0] = v
...     return 1

>>> from scipy import ndimage as ndi
>>> %timeit ndi.grey_erosion(image, footprint=footprint)
10 loops, best of 3: 118 ms per loop
>>> %timeit ndi.generic_filter(image, np.min, footprint=footprint)
1 loop, best of 3: 27 s per loop

>>> %timeit ndi.generic_filter(image, LowLevelCallable(nbmin.ctypes), footprint=footprint)
10 loops, best of 3: 113 ms per loop

import numpy as np

def addarr(x, y):
    result = np.zeros_like(x)
    for i in range(x.size):
        result[i] = x[i] + y[i]
    return result

n = int(1e6)
a = np.random.rand(n)
b = np.random.rand(n)

%timeit -r 1 -n 1 addarr(a, b)

1 loop, best of 1: 721 ms per loop

import numba

addarr_nb = numba.jit(addarr)

%timeit -r 1 -n 1 addarr_nb(a, b)

1 loop, best of 1: 283 ms per loop

%timeit -r 1 -n 1 addarr_nb(a, b)

1 loop, best of 1: 6.36 ms per loop

%timeit -r 1 -n 1 a + b

1 loop, best of 1: 5.62 ms per loop

r = np.random.randint(0, 128, size=n).astype(np.uint8)
s = np.random.randint(0, 128, size=n).astype(np.uint8)

%timeit -r 1 -n 1 r + s

1 loop, best of 1: 2.92 ms per loop

%timeit -r 1 -n 1 addarr_nb(r, s)

1 loop, best of 1: 238 ms per loop

%timeit -r 1 -n 1 addarr_nb(r, s)

1 loop, best of 1: 234 µs per loop

%matplotlib inline

import matplotlib.pyplot as plt
plt.rcParams['image.cmap'] = 'gray'
plt.rcParams['image.interpolation'] = 'nearest'

skeleton = np.array([[0, 1, 0, 0, 0, 1, 1],
                     [0, 0, 1, 1, 1, 0, 0],
                     [0, 1, 0, 0, 0, 1, 0],
                     [0, 0, 1, 0, 1, 0, 0],
                     [1, 1, 0, 1, 0, 0, 0]], dtype=bool)
skeleton = np.pad(skeleton, pad_width=1, mode='constant')

fig, ax = plt.subplots(figsize=(5, 5))
ax.imshow(skeleton)
ax.set_title('Skeleton')
ax.set_xlabel('col')
ax.set_ylabel('row')


from scipy import ndimage as ndi

neighbors = np.array([[1, 1, 1],
                      [1, 0, 1],
                      [1, 1, 1]])

degrees = ndi.convolve(skeleton.astype(int), neighbors) * skeleton

fig, ax = plt.subplots(figsize=(5, 5))
result = ax.imshow(degrees, cmap='magma')
ax.set_title('Skeleton, colored by node degree')
ax.set_xlabel('col')
ax.set_ylabel('row')
cbar = fig.colorbar(result, ax=ax, shrink=0.7)
cbar.set_ticks([0, 1, 2, 3])
cbar.set_label('degree')

fig, ax = plt.subplots(figsize=(5, 5))
result = ax.imshow(degrees, cmap='magma')
cbar = fig.colorbar(result, ax=ax, shrink=0.7)
cbar.set_ticks([0, 1, 2, 3])

nnz = len(np.flatnonzero(degrees))
pixel_labels = np.arange(nnz) + 1
for lab, y, x in zip(pixel_labels, *np.nonzero(degrees)):
    ax.text(x, y, lab, horizontalalignment='center',
            verticalalignment='center')

ax.set_xlabel('col')
ax.set_ylabel('row')
ax.set_title('Skeleton, with pixel IDs')
cbar.set_label('degree')

n_edges = np.sum(degrees)
row = np.empty(n_edges, dtype=np.int32)  # type expected by scipy.sparse
col = np.empty(n_edges, dtype=np.int32)

def neighbour_steps(shape):
    step_sizes = np.cumprod((1,) + shape[-1:0:-1])
    axis_steps = np.array([[-1, -1],
                           [-1,  1],
                           [ 1, -1],
                           [ 1,  1]])
    diag = axis_steps @ step_sizes
    steps = np.concatenate((step_sizes, -step_sizes, diag))
    return steps

steps = neighbour_steps(degrees.shape)
print(steps)

[  1   9  -1  -9 -10   8  -8  10]

def build_graph(labeled_pixels, steps_to_neighbours, row, col):
    start = np.max(steps_to_neighbours)
    end = len(labeled_pixels) - start
    elem = 0  # row/col index
    for k in range(start, end):
        i = labeled_pixels[k]
        if i != 0:
            for s in steps:
                neighbour = k + s
                j = labeled_pixels[neighbour]
                if j != 0:
                    row[elem] = i
                    col[elem] = j
                    elem += 1

skeleton_int = np.ravel(skeleton.astype(np.int32))
skeleton_int[np.nonzero(skeleton_int)] = 1 + np.arange(nnz)

%timeit -r 1 -n 1 build_graph(skeleton_int, steps, row, col)

1 loop, best of 1: 917 µs per loop

build_graph_nb = numba.jit(build_graph)

%timeit -r 1 -n 1 build_graph_nb(skeleton_int, steps, row, col)

1 loop, best of 1: 346 ms per loop

%timeit -r 1 -n 1 build_graph_nb(skeleton_int, steps, row, col)

1 loop, best of 1: 14.3 µs per loop

from scipy import sparse
G = sparse.coo_matrix((np.ones_like(row), (row, col))).tocsr()

import networkx as nx

Gnx = nx.from_scipy_sparse_matrix(G)
Gnx.remove_node(0)

nx.draw_spectral(Gnx, with_labels=True)

I Love Symposia! (Posts about open-source)

Why citations are not enough for open source software

How to fund open source

Why you should cite open source tools

Summer school announcement: 2nd Advanced Scientific Programming in Python (ASPP) Asia Pacific!

Key details

Background

Course description

Who is eligible?

More questions

The road to scikit-image 1.0

Part one: where we are

Part two: where we're going

Our vision

Our mission

Our values

Part three: scikit-image 1.0

The human side

What do scientists know about open source?

Scientific culture is ridiculously secretive

For coding, scientists have even more severe impostor syndrome

Scientists don't know about open source

What can scientists in open source do?

Prettier LowLevelCallables with Numba JIT and decorators

SciPy's new LowLevelCallable is a game-changer

Numba in the real world

Part 1: some toy examples¶

Part 2: Real Numba¶

Conclusion¶

Why scientists should code in the open

No, you won't get scooped

It's never going to be ready

You will gain collaborators and citations

Your coding ability will automagically improve

The SciPy ecosystem belongs to everyone