Howdy, Stranger!

It looks like you're new here. If you want to get involved, click one of these buttons!

Magic mirror/green screen type effect (with video)

edited August 2016 in Code Sharing Posts: 737

Messing around with the camera input - made a small program which acts a bit like the "green screen" visual effect used in (older) TV/films.

Best to use with a stationary iPad - ideally on a stand. Tap the bottom right, but without standing in the field of view of the camera to capture the reference background then walk in front of the camera.a__

-- Magic Mirror
supportedOrientations(LANDSCAPE_ANY)
displayMode(FULLSCREEN)
-- Use this function to perform your initial setup
function setup()
    counter=0
    cameraSource(CAMERA_FRONT)
    refimg = image(CAMERA)  --used as the background reference image
    curimg=image(CAMERA)  --used to store the current image from the camera

    xres=32/4  --the size of the horizontal blocks - should divide into 1280 to leave an integer
    yres=18/2  --the size of the vertical blocks - should divide into 720 to leave an integer
    thresh=40. -- a threshold for detecting the difference between the background pixel and current pixel
    dispimg=image(1280,720)  -- used for the image to be displayed
    bgimg=image(xres,yres) -- create a small image for use as the background blocks

    setContext(bgimg)
    background(0,0,200) -- currently set to blue but could be changed to anything
    setContext()


    --an alternative single background image.  make the code substitution commemted on later too
    --[[
    backimg=image(1280,720)
    setContext(backimg)
    sprite("Cargo Bot:Codea Icon",WIDTH/2,HEIGHT/2,1280,720)
    setContext()
    ]]--
end

-- This function gets called once every frame
function draw()

    counter = counter + 1
    text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
    if refimg~=nil then
        --
        if counter>0 then
            counter=0
            local curimg = image(CAMERA)
            setContext(dispimg)
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                for j=1,720,yres do
                    yc=yc+1
                    local r,g,b,a= curimg:get(i,j)
                    --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                    if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                        sprite(bgimg,i,j,xres,yres)
                        --substitute above with the following for a full single image as defined earlier
                        --  sprite(backimg:copy(i,j,xres,yres),i,j,xres,yres)
                    else
                        sprite(curimg:copy(i,j,xres,yres),i,j,xres,yres)
                    end
                end
            end

        end

        setContext()
        sprite(dispimg,WIDTH/2,HEIGHT/2)
    end
    --camera button image
    sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
    sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
    collectgarbage()
end

function touched(t)

    if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
        --take new reference image
        cameraSource(CAMERA_FRONT)
        refimg = image(CAMERA)
        --camera snapshot resolution from my instance is 1280,720
        --generate a series of checkpoints and store the pixel color for comparison
        ref={}
        local xc=0
        for i=1,1280,xres do
            local yc=0
            xc = xc + 1
            ref[xc]={}
            for j=1,720,yres do
                local red,green,blue,alpha= refimg:get(i,j)
                table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha})
            end
        end
    end
end

Comments

  • AnatolyAnatoly Mod
    Posts: 842

    So laggy :( But nice ;)

  • Posts: 737
    Yeah I'm looking at improving this
  • Posts: 160

    I tried it and it is super cool!

  • Posts: 160

    Here are some things that you could change in your code to make it run better:
    -Try using meshes instead of sprites.
    -Maybe you could make the camera image smaller (remove some pixels) so that there are less pixels to look at.

  • Posts: 737
    I've got a version which works a lot faster pasting the current image and pasting over the "mask" for unchanged background parts. However I'm having a problem with the positions at the moment
  • Posts: 737

    Here's a much faster version:

    -- Magic Mirror
    supportedOrientations(LANDSCAPE_ANY)
    displayMode(FULLSCREEN)
    -- Use this function to perform your initial setup
    function setup()
    
    
        counter=0
        cameraSource(CAMERA_FRONT)
    
        refimg = image(CAMERA)  --used as the background reference image
        curimg=image(CAMERA)  --used to store the current image from the camera
    
        xres=4  --the size of the horizontal blocks - should divide into 1280 to leave an integer
        yres=6  --the size of the vertical blocks - should divide into 720 to leave an integer
        thresh=60 -- a threshold for detecting the difference between the background pixel and current pixel
        dispimg=image(1280,720)  -- used for the image to be displayed
        bgimg=image(xres,yres) -- create a small image for use as the background blocks
    
        setContext(bgimg)
        background(0,0,200) -- currently set to blue but could be changed to anything
        setContext()
    
    end
    
    -- This function gets called once every frame
    function draw()
    
        counter = counter + 1
        text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
        if refimg~=nil then
            --
            if counter>0 then
                counter=0
                local curimg = image(CAMERA)
                setContext(dispimg)
                sprite(curimg,WIDTH/2+128,HEIGHT/2-24,1280,720)
                local xc=0
                for i=1,1280,xres do
                    local yc=0
                    xc = xc + 1
                    for j=1,720,yres do
                        yc=yc+1
                        local r,g,b,a= curimg:get(i,j)
                        --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                        if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                            --matches the reference so blank it out
                            sprite(bgimg,i-xres/2,j-yres/2,xres,yres)
    
                        else
    
                        end
                    end
                end
    
            end
    
            setContext()
            sprite(dispimg,WIDTH/2,HEIGHT/2,1280,720)
        end
        --camera button image
        sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
        sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
        collectgarbage()
    
    end
    
    function touched(t)
    
        if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
            --take new reference image
            cameraSource(CAMERA_FRONT)
            refimg = image(CAMERA)
            --camera snapshot resolution from my instance is 1280,720
            --generate a series of checkpoints and store the pixel color for comparison
            ref={}
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                ref[xc]={}
                for j=1,720,yres do
                    local red,green,blue,alpha= refimg:get(i,j)
                    table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha})
                end
            end
        end
    end
    
    
  • Posts: 737
    Some modifications to recognise specific colours. Here's a video

  • Posts: 160

    Wow! That's really cool looking!

  • Posts: 737
    @Kolosso thanks - pretty pleased at how it's turning out
  • Posts: 2,020
    That is awesome, what a great idea. A shader might help here
  • Posts: 737

    @yojimbo2000 I wondered that - I still haven't made time to learn shaders.

    Meanwhile here is the updated code which generates "fire" based on any orange object on screen. I've also modified to allow a background image. Could be the basis of one of these cheap trick pictures where you can superimpose yourself in a scene with the pyramids of Giza/Mount Rushmore/the moon landing/etc

    -- Magic Mirror
    supportedOrientations(LANDSCAPE_ANY)
    displayMode(FULLSCREEN)
    -- Use this function to perform your initial setup
    function setup()
        ref={}
        cameraSource(CAMERA_FRONT)
        refimg = image(CAMERA)  --used as the background reference image
        curimg=image(CAMERA)  --used to store the current image from the camera
        hearts={}
        xres=4--the size of the horizontal blocks - should divide into 1280 to leave an integer
        yres=6  --the size of the vertical blocks - should divide into 720 to leave an integer
        thresh=25-- a threshold for detecting the difference between the background pixel and current pixel
        dispimg=image(1280,720)  -- used for the image to be displayed
        bgimg=image(xres,yres) -- create a small image for use as the background blocks
        setContext(bgimg)
        background(72, 50, 225, 255) -- doesn't matter what color as this will be set as the transparent mask to background
        setContext()
    end
    
    -- This function gets called once every frame
    function draw()
        background(0)
        text("Tap in bottom right to set the background reference image",WIDTH/2,HEIGHT/2)
        sprite("SpaceCute:Background",WIDTH/2,HEIGHT/2,WIDTH,HEIGHT)
        if refimg~=nil then
            local curimg = image(CAMERA)
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                for j=1,720,yres do
                    yc=yc+1
                    local r,g,b,a= curimg:get(i,j)
                    --check each reference point in turn - if the rgb values are comparable to the reference image then treat as background, otherwise treat as new and show that portion of the camera image
                    if math.abs(ref[xc][yc].r-tonumber(r))<thresh and math.abs(ref[xc][yc].g-tonumber(g))<thresh and math.abs(ref[xc][yc].b-tonumber(b))<thresh then
                        -- set this block marker, m to zero (background)
                        ref[xc][yc].m=0
                    else
                        -- set this block marker, m to 1 (active)
                        ref[xc][yc].m=1
                        --check if the block is a certain colour - in this case orange.  if so the mark m as 2 (fire activater)
                        if r>220 and g>30 and g<230 and b<100 then
                            ref[xc][yc].m=2
    
                            if math.random(10)==1 then
                                table.insert(hearts,{x=i-128,y=j+24,fade=255,active=1,spd=2+math.random(10),off=math.random(10),size=10+math.random(15),wob=-5+math.random(10)})
                            end
                        end
                    end
                end
            end
    
            --eliminate single pixel noise
            for i=2,#ref-1 do
                for j=2,#ref[i]-1 do
                    if ref[i][j].m>0 and
                    ref[i-1][j+1].m==0 and ref[i][j+1].m==0 and ref[i+1][j+1].m==0 and
                    ref[i-1][j].m==0 and ref[i+1][j].m==0 and
                    ref[i-1][j-1].m==0 and ref[i][j-1].m==0 and ref[i+1][j-1].m==0 then
    
                        ref[i][j].m=0
                    end
                end
            end
    
            setContext(dispimg)
            sprite(curimg,WIDTH/2+128,HEIGHT/2-24,1280,720)
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                for j=1,720,yres do
                    yc=yc+1
                    if ref[xc][yc].m==0 then
                        --mask away current block
                        blendMode( ZERO, ONE_MINUS_SRC_ALPHA )
                        sprite(bgimg,i+xres/2,j+yres/2,xres,yres)
                        blendMode(NORMAL)
                    else
                        --leave current image portion to show through
                    end
                end
            end
            setContext()
            sprite(dispimg,WIDTH/2,HEIGHT/2)
        end
        --camera button image
        sprite("Cargo Bot:Crate Goal Yellow",WIDTH*0.95,HEIGHT*0.05,0.1*WIDTH,0.1*HEIGHT)
        sprite("Cargo Bot:Record Solution Icon",WIDTH*0.95,HEIGHT*0.05,0.05*WIDTH,0.05*HEIGHT)
    
        for i,h in pairs(hearts) do
            tint(255,h.fade)
            if h.fade>200 then
                sprite("Cargo Bot:Star Filled",h.x,h.y,h.size)
            else
                sprite("Cargo Bot:Smoke Particle",h.x,h.y,h.size)
    
            end
    
            h.y = h.y + h.spd
            if h.fade<200 then h.y = h.y + h.spd end
            h.x = h.x + h.wob*math.sin(ElapsedTime+h.off)
            h.fade = h.fade - 5
            if h.fade<160 then h.fade = h.fade - 5 end
            if h.fade<0 then h.active=0 end
    
        end
        noTint()
        for i=#hearts,1,-1 do
            if hearts[i].active==0 then
                table.remove(hearts,i)
            end
        end
        collectgarbage()
    end
    
    function touched(t)
    
        if t.state==ENDED and t.x>0.9*WIDTH and t.y<0.1*HEIGHT then
            --take new reference image
            cameraSource(CAMERA_FRONT)
            refimg = image(CAMERA)
            --camera snapshot resolution from my instance is 1280,720
            --generate a series of checkpoints and store the pixel color for comparison
    
            local xc=0
            for i=1,1280,xres do
                local yc=0
                xc = xc + 1
                ref[xc]={}
                for j=1,720,yres do
                    local red,green,blue,alpha= refimg:get(i,j)
                    table.insert(ref[xc],{r=red,g=green,b=blue,a=alpha,m=0})
                end
            end
        end
    end
    
    
Sign In or Register to comment.