Howdy, Stranger!

It looks like you're new here. If you want to get involved, click one of these buttons!

Another Free-to-Use Shader (Whoohoo!) Green Screen

in Shaders Posts: 160

Yesterday I decided to try to make a small project of a green screen for the fun of it and it reminded me a bit about (this project)[https://codea.io/talk/discussion/7634/magic-mirror-green-screen-type-effect-with-video] @West made a year ago. To use it, give it a color (the colour to detect) and two images (the camera and the replace texture). It will then draw the camera while replacing any pixels of similar colours to the supplied colour with the replace texture. You can also adjust the sensitivity of the shader by changing m.shader.sens. setting it to 0 will not replace anything and 1 will replace all pixels. I like it at 0.2 but you can change it at anytime. The shader isn't as good as @West's, but it's better than I thought it would be.

Below is an example project that uses the shader. You can modify it, learn from it, or add things to it.

--# Main

function setup()
    info = ""
    infoTimer = 0
    sampling = true
    cameraSource(CAMERA_FRONT)
    m = mesh()
    m:addRect(WIDTH/2,HEIGHT/2,0,0)
    m.texture = CAMERA -- 1920 x 1080
    m.shader = shader(Ghost.v,Ghost.f)
    m.shader.inactive = true
    parameter.boolean("Camera_Front",false,function(b)
        cameraSource((b and CAMERA_FRONT) or CAMERA_BACK)
    end)

    -- Hooray! Variables for playing with! : )
    replaceTexture = readImage("SpaceCute:Background") -- The texture that will replace certain pixels. (Will be stretched to fit the screen)
    sensitivity = 0.2 -- How different a pixels colour can be from the sample colour before being replaced
end

function draw()
    background(40, 40, 50)
    camW,camH = spriteSize(CAMERA)
    if CurrentOrientation == LANDSCAPE_LEFT or CurrentOrientation == LANDSCAPE_RIGHT then
        m:setRect(1,0,0,camW*(WIDTH/camW),camH*(WIDTH/camW))
        else
        m:setRect(1,0,0,camW*(HEIGHT/camH),camH*(HEIGHT/camH))
    end
    translate(WIDTH/2,HEIGHT/2)
    noSmooth()
    m:draw()
    if sampling then
        stroke(255, 255, 255, 255)
        strokeWidth(5)
        if sampleC then -- sampleC is the variable that holds the sampled colour
            fill(sampleC)
            else
            noFill()
        end
        ellipse(0,0,30)
        line(-110,0,-10,0)
        line(10,0,110,0)
        line(0,-110,0,-10)
        line(0,10,0,110)
        fill(255)
        fontSize(20)
        smooth()
        text("Tap the center of the screen to sample the colour",0,HEIGHT/2 - 30)
        text("Tap outside of the center to activate the shader",0,HEIGHT/-2 + 30)
        else
        smooth()
        text("Tap anywhere to return to the colour sampler",0,HEIGHT/-2 + 30)
    end
    if infoTimer > 0 then -- This is responsible for the blue text that appears in the center of the screen.
        infoTimer = infoTimer - DeltaTime
        fill(0,200,255)
        fontSize(30)
        text(info,0,-200)
    end
end

function touched(t)
    if t.state == 0 then
        if sampling and ((t.x - WIDTH/2)^2 + (t.y - HEIGHT/2)^2)^0.5 < 180 then
            if not sampleC then
                info = "This colour will now be used in the shader"
                infoTimer = 4
            end

            local samCam = image(WIDTH,HEIGHT)
            pushStyle()
            smooth()
            setContext(samCam)
            m:draw()
            setContext()
            popStyle()
            local all = {r=0,g=0,b=0,c=0}
            for x = WIDTH//2 - 10, WIDTH//2 + 10 do
                for y = HEIGHT//2 - 10, HEIGHT//2 + 10 do
                    if ((x - WIDTH//2)^2 + (y - HEIGHT//2)^2)^0.5 < 10 then--if this pixel is in a 10 pixel radius
                        local r,g,b = samCam:get(x,y)
                        all.r = all.r + r
                        all.g = all.g + g
                        all.b = all.b + b
                        all.c = all.c + 1 
                    end
                end
            end
            all.r,all.g,all.b = all.r/all.c,all.g/all.c,all.b/all.c
            sampleC = color(all.r,all.g,all.b,255)

            samCam = nil
            all = nil
            collectgarbage()
            elseif sampleC then
            sampling = not sampling
            if sampling then
                m.shader.inactive = true
                else
                m.shader.sens = sensitivity
                m.shader.inactive = false
                m.shader.overTex = replaceTexture
                m.shader.sample = sampleC
            end
            else
            info = "You need to sample a colour for the shader, first"
            infoTimer = 4
        end
    end
end

Ghost = {
v = [[uniform mat4 modelViewProjection;
attribute vec4 position;
attribute vec4 color;
attribute vec2 texCoord;
varying lowp vec4 vColor;
varying highp vec2 vTexCoord;
void main() { vColor = color; vTexCoord = texCoord; gl_Position = modelViewProjection * position; }]],
f = [[
precision highp float;

uniform lowp float sens;
uniform lowp sampler2D texture;
uniform lowp sampler2D overTex;
uniform lowp vec4 sample;
uniform bool inactive;

varying lowp vec4 vColor;
varying highp vec2 vTexCoord;

void main()
{
    lowp vec4 col = texture2D( texture, vTexCoord );
    if (inactive == false && col.r > sample.r - sens && col.r < sample.r + sens && col.g > sample.g - sens && col.g < sample.g + sens && col.b < sample.b + sens && col.b > sample.b - sens) {
        col = texture2D( overTex, vTexCoord );
    }
    gl_FragColor = col;
}
]]
}
Tagged:

Comments

  • dave1707dave1707 Mod
    Posts: 7,553

    @Kolosso Great example. It will be fun to play with.

  • Posts: 160

    @dave1707 Thanks
    I believe I thought of an algorithm that could make the green screen work much better but I'm not sure how to make it into a shader, and I fear that if I don't make it as a shader it'll be way too slow. I'm gonna try doing it myself but if I can't do it, I'll try to get some help

  • dave1707dave1707 Mod
    edited August 2017 Posts: 7,553

    @Kolosso Thought I'd try your green screen example. Put the red circle that's in the center of the screen over the color to replace and tap the screen. Use the color parameter to select the replacement color.

    EDIT: Added the parameter range.

    displayMode(OVERLAY)
    
    function setup()
        parameter.color("c1",255,0,0,change)
        parameter.number("range",0.0,1.0,.2,change)
        m=mesh() 
        m:addRect(WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
    end
    
    function draw()
        background(0)
        img=image(CAMERA)
        m.texture=img
        m:draw()
        collectgarbage()
        noFill()
        stroke(255,0,0)
        strokeWidth(2)
        ellipse(WIDTH/2,HEIGHT/2,20)
    end
    
    function change()
        if img~=nil and m.shader~=nil then
            m.shader.colOut=vec4(c1.r/256,c1.g/256,c1.b/256,a/256)
            m.shader.range=range
        end
    end
    
    function touched(t)
        if t.state==BEGAN then
            m.shader=shader(sh.v,sh.f)
            r,g,b,a=img:get(img.width//2,img.height//2)
            m.shader.colIn=vec4(r/256,g/256,b/256,a/256)
            m.shader.colOut=vec4(c1.r/256,c1.g/256,c1.b/256,c1.a/256)
            m.shader.range=range
        end
    end
    
    sh={
        v=[[
        uniform mat4 modelViewProjection;
        attribute vec4 position;
        attribute vec4 color;
        attribute vec2 texCoord;
        varying lowp vec4 vColor;
        varying highp vec2 vTexCoord;
        void main()
        {   vColor = color;
            vTexCoord = texCoord;
            gl_Position = modelViewProjection * position;
        }   ]],
    
        f=[[
        precision highp float;
        uniform lowp vec4 colIn;
        uniform lowp vec4 colOut;
        uniform lowp float range;
        uniform lowp sampler2D texture;
        varying lowp vec4 vColor;
        varying highp vec2 vTexCoord;
        void main()
        {   lowp vec4 col = texture2D( texture, vTexCoord ) * vColor;
            if (abs(colIn.r-col.r)<range && abs(colIn.g-col.g)<range && 
                    abs(colIn.b-col.b)<range && abs(colIn.a-col.a)<range)
            {   col.r = colOut.r;
                col.g = colOut.g;
                col.b = colOut.b;
            }
            gl_FragColor = col;
        }
        ]]
        }
    
  • Posts: 160

    @dave1707 Wow that's neat! Although it seems a bit slower than my project. (probably because of the line img=image(CAMERA)) I didn't work on improving my green screen shader yet because I was too busy relaxing and playing the classic PS2. I'll do it tomorrow.

Sign In or Register to comment.