Demo

Explanation

I wanted the video's uv coords to follow a discrete fluid PDE I got from a physics college (That I do not understand). I don't think I converted the math right or the outcome was just boring. Anyway I changed the noise function to vary with time and got back a better distortion. It just ends with the UVs begin shifted in one direction and a noisy image. After some time there are interesting patterns though.

\psi^{t+1}_{xy} = \Delta t \left( V^t_x \frac{\psi^t_{x-1 y} - \psi^t_{x+1 y}}{2 \Delta x} + V^t_y \frac{\psi^t_{xy-1} - \psi^t_{xy+1}}{2 \Delta y} \right) + \psi^t_{xy} \\ \text{where } V \text{ is the noise function and } \psi \text{ is the uv cord.}

I wanted to setup and use multiple buffers as you would have in shadertoy. I first prototyped in shadertoy (webgpu broke alot, but gulls is cool). Finally I just wanted to add more uniforms, so I though a simple window to view the original video would work. Most functions were translated/copied from iquilezles, book of shaders and is gist for wgsl noise functions.

Notes: I probably would have done more if webgpu had not been a problematic. Firefox's webgpu is broken with textures. Vulkan on NVIDIA is also annoying. (like it is with wgpu-native). I should have used compute.toys for prototyping.

Reaction

I was told the performance was pretty bad by the college who reviewed it and got some advice on how to fix it (unfortunately it required gutting out gulls). Other than that, the effect was okay. The circle interaction was interesting.

I honestly felt the same that the uv coord buffer was not that interesting. But there is at least the framework setup for anything that involves similar transformation of a texture's uv coords.

Code

uv shader
@group(0) @binding(0) var<uniform> u_res: vec2f;
@group(0) @binding(1) var<uniform> u_frame: f32;
@group(0) @binding(2) var<uniform> u_mouse: vec3f;

@group(0) @binding(3) var<uniform> u_background: vec3f;

@group(0) @binding(4) var videoSampler: sampler;
@group(0) @binding(5) var backBuffer: texture_2d<f32>;

fn random(st: vec2f) -> f32 { return fract(sin(dot(st.xy, vec2(12.9898, 78.233))) * 43758.5453123); }
fn random2(p: vec2f) -> vec2f { return fract(sin(vec2(dot(p, vec2(127.1, 311.7)), dot(p, vec2(269.5, 183.3)))) * 43758.5453); }

// 2D Noise based on Morgan McGuire @morgan3d
// https://www.shadertoy.com/view/4dS3Wd
fn noise(st: vec2f) -> f32 {
    let i = floor(st);
    let f = fract(st);
    let a = random(i);
    let b = random(i + vec2(1.0, 0.0));
    let c = random(i + vec2(0.0, 1.0));
    let d = random(i + vec2(1.0, 1.0));
    let u = f * f * (3.0 - 2.0 * f);
    return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y;
}

// MIT License. © Stefan Gustavson, Munrocket
//
fn permute4(x: vec4f) -> vec4f { return ((x * 34. + 1.) * x) % vec4f(289.); }
fn fade2(t: vec2f) -> vec2f { return t * t * t * (t * (t * 6. - 15.) + 10.); }

fn perlinNoise2(P: vec2f) -> f32 {
    var Pi: vec4f = floor(P.xyxy) + vec4f(0., 0., 1., 1.);
    let Pf = fract(P.xyxy) - vec4f(0., 0., 1., 1.);
    Pi = Pi % vec4f(289.); // To avoid truncation effects in permutation
    let ix = Pi.xzxz;
    let iy = Pi.yyww;
    let fx = Pf.xzxz;
    let fy = Pf.yyww;
    let i = permute4(permute4(ix) + iy);
    var gx: vec4f = 2. * fract(i * 0.0243902439) - 1.; // 1/41 = 0.024...
    let gy = abs(gx) - 0.5;
    let tx = floor(gx + 0.5);
    gx = gx - tx;
    var g00: vec2f = vec2f(gx.x, gy.x);
    var g10: vec2f = vec2f(gx.y, gy.y);
    var g01: vec2f = vec2f(gx.z, gy.z);
    var g11: vec2f = vec2f(gx.w, gy.w);
    let norm = 1.79284291400159 - 0.85373472095314 *
        vec4f(dot(g00, g00), dot(g01, g01), dot(g10, g10), dot(g11, g11));
    g00 = g00 * norm.x;
    g01 = g01 * norm.y;
    g10 = g10 * norm.z;
    g11 = g11 * norm.w;
    let n00 = dot(g00, vec2f(fx.x, fy.x));
    let n10 = dot(g10, vec2f(fx.y, fy.y));
    let n01 = dot(g01, vec2f(fx.z, fy.z));
    let n11 = dot(g11, vec2f(fx.w, fy.w));
    let fade_xy = fade2(Pf.xy);
    let n_x = mix(vec2f(n00, n01), vec2f(n10, n11), vec2f(fade_xy.x));
    let n_xy = mix(n_x.x, n_x.y, fade_xy.y);
    return 2.3 * n_xy;
}

@fragment
fn fs(@builtin(position) pos: vec4f) -> @location(0) vec4f {
    let uv = pos.xy / u_res;

    if u_frame <= 100.0 || (u_mouse.z > 0.0) {
        return vec4f(uv, 0.0, 1.0);
    }

    let dt = 1. / 60.;
    let time = u_frame / 60.;

    let dx = vec2(0.4, 0.9);
    let dy = vec2(0.5, 0.7);

    let Ixy = pos.xy;

    let Ix1y = pos.xy + vec2(1., 0.);
    let Ix_1y = pos.xy - vec2(1., 0.);

    let Ixy1 = pos.xy + vec2(0., 1.);
    let Ixy_1 = pos.xy - vec2(0., 1.);

    // let Vxy = vec2(perlinNoise2((pos.xy + sin(u_frame)) * 50.));
    // let Vxy = 50. * vec2(-1, -1) * (vec2(2.0 * perlinNoise2(pos.xy / 30. + vec2(10.0 * time))) - 1.);
    let Vxy = 50. * random2(uv) * (vec2(2.0 * perlinNoise2(pos.xy / 30. + vec2(10.0 * time)))
        - 1.);

    // let Vxy = vec2(perlinNoise2(pos.xy * 10.));

    let Ixyt1 = dt * (vec2(0.) +
                Vxy.x * (Ix1y - Ix_1y) / (2.0 * dx) +
                Vxy.y * (Ixy1 - Ixy_1) / (2.0 * dy)) + Ixy;

    var color = textureSample(backBuffer, videoSampler, Ixyt1 / u_res.xy).xyz;

    return vec4f(color, 1.0);
}
image shader
@group(0) @binding(0) var<uniform> u_res: vec2f;
@group(0) @binding(1) var<uniform> u_frame: f32;
@group(0) @binding(2) var<uniform> u_mouse: vec3f;

@group(0) @binding(3) var<uniform> u_background: vec3f;
@group(0) @binding(4) var<uniform> u_show_uv_buffer: f32;
@group(0) @binding(5) var<uniform> u_radius: f32;
@group(0) @binding(6) var<uniform> u_border: f32;

@group(0) @binding(7) var videoSampler: sampler;
@group(0) @binding(8) var uvBuffer: texture_2d<f32>;

@group(1) @binding(0) var videoBuffer: texture_external;

fn random(st: vec2f) -> f32 { return fract(sin(dot(st.xy, vec2(12.9898, 78.233))) * 43758.5453123); }
fn random2(p: vec2f) -> vec2f { return fract(sin(vec2(dot(p, vec2(127.1, 311.7)), dot(p, vec2(269.5, 183.3)))) * 43758.5453); }

// 2D Noise based on Morgan McGuire @morgan3d
// https://www.shadertoy.com/view/4dS3Wd
fn noise(st: vec2f) -> f32 {
    let i = floor(st);
    let f = fract(st);
    let a = random(i);
    let b = random(i + vec2(1.0, 0.0));
    let c = random(i + vec2(0.0, 1.0));
    let d = random(i + vec2(1.0, 1.0));
    let u = f * f * (3.0 - 2.0 * f);
    return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y;
}

// MIT License. © Stefan Gustavson, Munrocket
//
fn permute4(x: vec4f) -> vec4f { return ((x * 34. + 1.) * x) % vec4f(289.); }
fn fade2(t: vec2f) -> vec2f { return t * t * t * (t * (t * 6. - 15.) + 10.); }

fn perlinNoise2(P: vec2f) -> f32 {
    var Pi: vec4f = floor(P.xyxy) + vec4f(0., 0., 1., 1.);
    let Pf = fract(P.xyxy) - vec4f(0., 0., 1., 1.);
    Pi = Pi % vec4f(289.); // To avoid truncation effects in permutation
    let ix = Pi.xzxz;
    let iy = Pi.yyww;
    let fx = Pf.xzxz;
    let fy = Pf.yyww;
    let i = permute4(permute4(ix) + iy);
    var gx: vec4f = 2. * fract(i * 0.0243902439) - 1.; // 1/41 = 0.024...
    let gy = abs(gx) - 0.5;
    let tx = floor(gx + 0.5);
    gx = gx - tx;
    var g00: vec2f = vec2f(gx.x, gy.x);
    var g10: vec2f = vec2f(gx.y, gy.y);
    var g01: vec2f = vec2f(gx.z, gy.z);
    var g11: vec2f = vec2f(gx.w, gy.w);
    let norm = 1.79284291400159 - 0.85373472095314 *
        vec4f(dot(g00, g00), dot(g01, g01), dot(g10, g10), dot(g11, g11));
    g00 = g00 * norm.x;
    g01 = g01 * norm.y;
    g10 = g10 * norm.z;
    g11 = g11 * norm.w;
    let n00 = dot(g00, vec2f(fx.x, fy.x));
    let n10 = dot(g10, vec2f(fx.y, fy.y));
    let n01 = dot(g01, vec2f(fx.z, fy.z));
    let n11 = dot(g11, vec2f(fx.w, fy.w));
    let fade_xy = fade2(Pf.xy);
    let n_x = mix(vec2f(n00, n01), vec2f(n10, n11), vec2f(fade_xy.x));
    let n_xy = mix(n_x.x, n_x.y, fade_xy.y);
    return 2.3 * n_xy;
}

@fragment
fn fs(@builtin(position) pos: vec4f) -> @location(0) vec4f {
    let u_scroll_speed = 0.0002;

    let uv = pos.xy / u_res;
    let fb = textureSample(uvBuffer, videoSampler, uv).xy;

    if u_show_uv_buffer > 0.5 {
        return vec4f(vec3(fb, 0.0), 1.0);
    }

    let distorted = textureSampleBaseClampToEdge(videoBuffer, videoSampler, fb).xyz;
    let normal = textureSampleBaseClampToEdge(videoBuffer, videoSampler, uv).xyz;

    let d = length(uv - u_mouse.xy);

    let n = noise((uv + u_frame * u_scroll_speed) * 50.0) * 0.02;

    let dn = d + n;

    let inside = 1.0 - smoothstep(u_radius - 0.002, u_radius + 0.002, dn);

    let inner = smoothstep(u_radius - u_border, u_radius, dn);
    let outer = smoothstep(u_radius, u_radius + u_border, dn);
    let border = inner - outer;

    var color = mix(distorted, normal, inside);

    color += u_background * border * 1.5;

    return vec4f(color, 1.0);
}
js
import { default as gulls } from "./gulls.js";
import { Pane } from "./tweakpane-4.0.5.min.js";
import { default as Video } from "./video.mjs";
import { default as Mouse } from "./mouse.mjs";
await Video.init();
Mouse.init();
const sg = await gulls.init();
const uvShader = gulls.constants.vertex + await gulls.import("./uv.frag.wgsl");
const imageShader = gulls.constants.vertex + await gulls.import("./image.frag.wgsl");
const pane = new Pane();
const u_res = sg.uniform([window.innerWidth, window.innerHeight]);
let u_frame = sg.uniform(0);
const u_mouse = sg.uniform([0, 0, 0]);
pane.addBinding(u_frame, 'value', {
    readonly: true,
    label: "frame"
});
const params = {
    background: { r: 194 / 255, g: 87 / 255, b: 180 / 255 },
    show_uv_buffer: false,
    radius: 0.1,
    border: 0.01,
};
const u_background = sg.uniform(Object.values(params.background));
pane
    .addBinding(params, "background", { color: { type: "float" } })
    .on("change", (_) => (u_background.value = Object.values(params.background)));
const u_show_uv_buffer = sg.uniform(0);
pane.addBinding(params, "show_uv_buffer").on("change", (_) => (params.show_uv_buffer ? (u_show_uv_buffer.value = 1.0) : (u_show_uv_buffer.value = 0.0)));
const u_radius = sg.uniform(0.1);
const u_border = sg.uniform(0.01);
pane.addBinding(params, "radius", { min: 0.1, max: 0.5 }).on("change", (_) => (u_radius.value = params.radius));
pane.addBinding(params, "border", { min: 0.01, max: 0.1 }).on("change", (_) => (u_border.value = params.border));
const back = new Float32Array(gulls.width * gulls.height * 4);
const feedback_t = sg.texture(back);
const uvRenderPass = await sg.render({
    shader: uvShader,
    data: [
        u_res,
        u_frame,
        u_mouse,
        u_background,
        sg.sampler(),
        feedback_t,
    ],
    copy: feedback_t,
    onframe() {
        u_frame.value++;
        u_mouse.value = Mouse.values;
        console.log(u_frame.value);
    },
});
const imageRenderPass = await sg.render({
    shader: imageShader,
    data: [
        u_res,
        u_frame,
        u_mouse,
        u_background,
        u_show_uv_buffer,
        u_radius,
        u_border,
        sg.sampler(),
        feedback_t,
        sg.video(Video.element),
    ],
});
sg.run(uvRenderPass, imageRenderPass);