How do you play video with alpha using AVFoundation?

Question

I have an AR application that uses SceneKitand imports a video into the scene using AVPlayerand thereby adds it as a child node from SKVideonode.

The video is visible as expected, but transparency in the video is not achieved.

Enter the code as follows:

let spriteKitScene = SKScene(size: CGSize(width: self.sceneView.frame.width, height: self.sceneView.frame.height))
spriteKitScene.scaleMode = .aspectFit

guard let fileURL = Bundle.main.url(forResource: "Triple_Tap_1", withExtension: "mp4") else {
    return
}

let videoPlayer = AVPlayer(url: fileURL)
videoPlayer.actionAtItemEnd = .none

let videoSpriteKitNode = SKVideoNode(avPlayer: videoPlayer)
videoSpriteKitNode.position = CGPoint(x: spriteKitScene.size.width / 2.0, y: spriteKitScene.size.height / 2.0)
videoSpriteKitNode.size = spriteKitScene.size
videoSpriteKitNode.yScale = -1.0
videoSpriteKitNode.play()
spriteKitScene.backgroundColor = .clear          
spriteKitScene.addChild(videoSpriteKitNode)

let background = SCNPlane(width: CGFloat(2), height: CGFloat(2))
background.firstMaterial?.diffuse.contents = spriteKitScene

let backgroundNode = SCNNode(geometry: background)
backgroundNode.position = position
backgroundNode.constraints = [SCNBillboardConstraint()]
backgroundNode.rotation.z = 0
self.sceneView.scene.rootNode.addChildNode(backgroundNode)

// Create a transform with a translation of 0.2 meters in front of the camera.
var translation = matrix_identity_float4x4
translation.columns.3.z = -0.2
let transform = simd_mul((self.session.currentFrame?.camera.transform)!, translation)

// Add a new anchor to the session.
let anchor = ARAnchor(transform: transform)
self.sceneView.session.add(anchor: anchor)

What could be the best way to implement video transparency Triple_Tap_1in this case. I looked at some stack overflow issues on this topic and found the only solution to be the KittyBoom ​​repository, which was created sometime in 2013 using Objective C.

I hope the community can uncover the best solution to this problem. GPUImagethe library cannot work.

Answer 1

. . Apple.

, .

---

1.

• , . = , = ( , , ).

• SKScene , , , material.transparent.contents ( , )

  let spriteKitOpaqueScene = SKScene(...)
  let spriteKitMaskScene = SKScene(...)
  ... // creating SKVideoNodes and AVPlayers for each video etc
  
  let material = SCNMaterial()
  material.diffuse.contents = spriteKitOpaqueScene
  material.transparent.contents = spriteKitMaskScene
  
  let background = SCNPlane(...)
  background.materials = [material]
  

• . "" (, , ) .

  let surfaceShader = "_surface.transparent.a = 1 - _surface.transparent.r;"
  material.shaderModifiers = [ .surface: surfaceShader ]
  

! .

, AVPlayer, , . , ( , ). .

:

• ( )
• Precise

:

• AVPlayer s

---

2.

• , , . .

• SKScene , , material.diffuse.contents.

• , . GPUImage, , . .

   let surfaceShader =
  """
  uniform vec3 c_colorToReplace = vec3(0, 1, 0);
  uniform float c_thresholdSensitivity = 0.05;
  uniform float c_smoothing = 0.0;
  
  #pragma transparent
  #pragma body
  
  vec3 textureColor = _surface.diffuse.rgb;
  
  float maskY = 0.2989 * c_colorToReplace.r + 0.5866 * c_colorToReplace.g + 0.1145 * c_colorToReplace.b;
  float maskCr = 0.7132 * (c_colorToReplace.r - maskY);
  float maskCb = 0.5647 * (c_colorToReplace.b - maskY);
  
  float Y = 0.2989 * textureColor.r + 0.5866 * textureColor.g + 0.1145 * textureColor.b;
  float Cr = 0.7132 * (textureColor.r - Y);
  float Cb = 0.5647 * (textureColor.b - Y);
  
  float blendValue = smoothstep(c_thresholdSensitivity, c_thresholdSensitivity + c_smoothing, distance(vec2(Cr, Cb), vec2(maskCr, maskCb)));
  
  float a = blendValue;
  _surface.transparent.a = a;
  """
  
  shaderModifiers = [ .surface: surfaceShader ]
  

  setValue(:forKey:).

  let vector = SCNVector3(x: 0, y: 1, z: 0) // represents float RGB components
  setValue(vector, forKey: "c_colorToReplace")
  setValue(0.3 as Float, forKey: "c_smoothing")
  setValue(0.1 as Float, forKey: "c_thresholdSensitivity")
  

  as Float , Swift Double, .

  , , - c_smoothing c_thresholdSensitivity. , , , .

:

• .

:

• ( )