Creating Unity VR Drawing App

by Jennifer Wang (2022)

Requirements

Step 1. Create an empty object called Mesh and add the components Mesh Filter and Mesh Renderer (select a material for element 0 as the color of the brush stroke). Add the script "Mesh" with the following code as a component to this object.

using System.Collections;

using System.Collections.Generic;

using UnityEngine;


public class BrushStrokeMesh : MonoBehaviour {

   [SerializeField]

   private float _brushStrokeWidth = 0.05f;


   private Mesh _mesh;


   private List<Vector3> _vertices;

   private List<Vector3> _normals;


   private bool _skipLastRibbonPoint;

   public  bool  skipLastRibbonPoint { get { return _skipLastRibbonPoint; } set { if (value == _skipLastRibbonPoint) return; _skipLastRibbonPoint = value; UpdateGeometry(); } }


   private void Awake() {

       MeshFilter filter = gameObject.GetComponent<MeshFilter>();

       _mesh = filter.mesh;


       _vertices = new List<Vector3>();

       _normals  = new List<Vector3>();


       // In addition to clearing the ribbon, this adds a ribbon point that we'll move each frame to match the

       // brush tip position so that the brush stroke appears to paint continuously.

       ClearRibbon();

   }


   // Insert a new ribbon point just before the final ribbon point

   public void InsertRibbonPoint(Vector3 position, Quaternion rotation) {

       // Calculate vertices + normal for ribbon point

       Vector3 p1;

       Vector3 p2;

       Vector3 normal;

       CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);


       // Insert into vertices array

       _vertices.Insert(_vertices.Count-4, p1);

       _vertices.Insert(_vertices.Count-4, p2);

       _vertices.Insert(_vertices.Count-4, p1);

       _vertices.Insert(_vertices.Count-4, p2);


       // Insert into normals array

       _normals.Insert(_normals.Count-4,  normal);

       _normals.Insert(_normals.Count-4,  normal);

       _normals.Insert(_normals.Count-4, -normal);

       _normals.Insert(_normals.Count-4, -normal);


       // Update the mesh

       UpdateGeometry();

   }


   public void UpdateLastRibbonPoint(Vector3 position, Quaternion rotation) {

       // Calculate vertices + normal for ribbon point

       Vector3 p1;

       Vector3 p2;

       Vector3 normal;

       CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);


       int lastIndex = _vertices.Count-4;


       // Update vertices

       _vertices[lastIndex]   = p1;

       _vertices[lastIndex+1] = p2;

       _vertices[lastIndex+2] = p1;

       _vertices[lastIndex+3] = p2;


       // Update normals

       _normals[lastIndex]   =  normal;

       _normals[lastIndex+1] =  normal;

       _normals[lastIndex+2] = -normal;

       _normals[lastIndex+3] = -normal;


       // Update the mesh

       UpdateGeometry();

   }


   public void ClearRibbon() {

       // Clear vertices & normals

       _vertices.Clear();

       _normals.Clear();


       // Create last ribbon point

       _vertices.Add(Vector3.zero);

       _vertices.Add(Vector3.zero);

       _vertices.Add(Vector3.zero);

       _vertices.Add(Vector3.zero);

       _normals.Add(Vector3.zero);

       _normals.Add(Vector3.zero);

       _normals.Add(Vector3.zero);

       _normals.Add(Vector3.zero);


       // Update the mesh

       UpdateGeometry();

   }


   private void CalculateVerticesAndNormalForRibbonPoint(Vector3 position, Quaternion rotation, float width, out Vector3 p1, out Vector3 p2, out Vector3 normal) {

       p1     = position + rotation * new Vector3(-width/2.0f, 0.0f, 0.0f);

       p2     = position + rotation * new Vector3( width/2.0f, 0.0f, 0.0f);

       normal = rotation * Vector3.up;

   }


   private void UpdateGeometry() {

       int numberOfVertices = _vertices.Count;

       if (skipLastRibbonPoint)

           numberOfVertices -= 4;


       if (numberOfVertices < 8) {

           _mesh.vertices  = new Vector3[0];

           _mesh.normals   = new Vector3[0];

           _mesh.triangles = new int[0];


           _mesh.RecalculateBounds();


           return;

       }


       // Would probably make sense to just generate the new triangles rather than regenerating all of them all of the time.


       int numberOfSegments  = numberOfVertices/4 - 1;

       int numberOfTriangles = numberOfSegments * 4; // Two on the front side, two on the back.


       int[] triangles = new int[numberOfTriangles*3];

       for (int i = 0; i < numberOfSegments; i++) {

           // Front

           int p1 = i*4;

           int p2 = i*4+1;

           int p3 = i*4+4;

           int p4 = i*4+5;


           // Back

           int p1b = i*4+2;

           int p2b = i*4+3;

           int p3b = i*4+6;

           int p4b = i*4+7;


           // Front

           triangles[i*12]   = p1;

           triangles[i*12+1] = p2;

           triangles[i*12+2] = p3;

           triangles[i*12+3] = p2;

           triangles[i*12+4] = p4;

           triangles[i*12+5] = p3;


           // Back

           triangles[i*12+6= p1b;

           triangles[i*12+7= p3b;

           triangles[i*12+8= p2b;

           triangles[i*12+9= p2b;

           triangles[i*12+10] = p3b;

           triangles[i*12+11] = p4b;

       }


       _mesh.vertices  = _vertices.ToArray();

       _mesh.normals   = _normals.ToArray();

       _mesh.triangles = triangles;


       _mesh.RecalculateBounds();

   }

}

Step 2. Create an empty object called BrushStroke and add the script "BrushStroke" with the following code as a component to this object.  Then add Mesh as a child to this object.

using System.Collections.Generic;

using UnityEngine;


public class BrushStroke : MonoBehaviour {

   [SerializeField]

   private BrushStrokeMesh _mesh = null;


   // Ribbon State

   struct RibbonPoint {

       public Vector3    position;

       public Quaternion rotation;

   }

   private List<RibbonPoint> _ribbonPoints = new List<RibbonPoint>();


   private Vector3    _brushTipPosition;

   private Quaternion _brushTipRotation;

   private bool       _brushStrokeFinalized;


   // Smoothing

   private Vector3    _ribbonEndPosition;

   private Quaternion _ribbonEndRotation = Quaternion.identity;


   // Mesh

   private Vector3    _previousRibbonPointPosition;

   private Quaternion _previousRibbonPointRotation = Quaternion.identity;


   // Unity Events

   private void Update() {

       // Animate the end of the ribbon towards the brush tip

       AnimateLastRibbonPointTowardsBrushTipPosition();


       // Add a ribbon segment if the end of the ribbon has moved far enough

       AddRibbonPointIfNeeded();

   }


   // Interface

   public void BeginBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {

       // Update the model

       _brushTipPosition = position;

       _brushTipRotation = rotation;


       // Update last ribbon point to match brush tip position & rotation

       _ribbonEndPosition = position;

       _ribbonEndRotation = rotation;

       _mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);

   }


   public void MoveBrushTipToPoint(Vector3 position, Quaternion rotation) {

       _brushTipPosition = position;

       _brushTipRotation = rotation;

   }


   public void EndBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {

       // Add a final ribbon point and mark the stroke as finalized

       AddRibbonPoint(position, rotation);

       _brushStrokeFinalized = true;

   }



   // Ribbon drawing

   private void AddRibbonPointIfNeeded() {

       // If the brush stroke is finalized, stop trying to add points to it.

       if (_brushStrokeFinalized)

           return;


       if (Vector3.Distance(_ribbonEndPosition, _previousRibbonPointPosition) >= 0.01f ||

           Quaternion.Angle(_ribbonEndRotation, _previousRibbonPointRotation) >= 10.0f) {


           // Add ribbon point model to ribbon points array. This will fire the RibbonPointAdded event to update the mesh.

           AddRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);


           // Store the ribbon point position & rotation for the next time we do this calculation

           _previousRibbonPointPosition = _ribbonEndPosition;

           _previousRibbonPointRotation = _ribbonEndRotation;

       }

   }


   private void AddRibbonPoint(Vector3 position, Quaternion rotation) {

       // Create the ribbon point

       RibbonPoint ribbonPoint = new RibbonPoint();

       ribbonPoint.position = position;

       ribbonPoint.rotation = rotation;

       _ribbonPoints.Add(ribbonPoint);


       // Update the mesh

       _mesh.InsertRibbonPoint(position, rotation);

   }


   // Brush tip + smoothing

   private void AnimateLastRibbonPointTowardsBrushTipPosition() {

       // If the brush stroke is finalized, skip the brush tip mesh, and stop animating the brush tip.

       if (_brushStrokeFinalized) {

           _mesh.skipLastRibbonPoint = true;

           return;

       }


       Vector3    brushTipPosition = _brushTipPosition;

       Quaternion brushTipRotation = _brushTipRotation;


       // If the end of the ribbon has reached the brush tip position, we can bail early.

       if (Vector3.Distance(_ribbonEndPosition, brushTipPosition) <= 0.0001f &&

           Quaternion.Angle(_ribbonEndRotation, brushTipRotation) <= 0.01f) {

           return;

       }


       // Move the end of the ribbon towards the brush tip position

       _ribbonEndPosition =     Vector3.Lerp(_ribbonEndPosition, brushTipPosition, 25.0f * Time.deltaTime);

       _ribbonEndRotation = Quaternion.Slerp(_ribbonEndRotation, brushTipRotation, 25.0f * Time.deltaTime);


       // Update the end of the ribbon mesh

       _mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);

   }

}



Step 3. Drag the object BrushStroke into assets to set it to a prefab. Delete the object from scene.

Step 4. Create an empty object called Brush and add the following script to its components.

using System.Collections.Generic;

using UnityEngine;

using UnityEngine.XR;

using UnityEngine.UI;


public class Brush : MonoBehaviour

{

   // Prefab to instantiate when we draw a new brush stroke

   [SerializeField] private GameObject _brushStrokePrefab = null;


   // Which hand should this brush instance track?

   private enum Hand { LeftHand, RightHand };

   [SerializeField] private Hand _hand = Hand.RightHand;

   // Used to keep track of the current brush tip position and the actively drawing brush stroke

   private Vector3 _handPosition;

   private Quaternion _handRotation;

   private BrushStroke _activeBrushStroke;

   public InputDeviceCharacteristics controllerCharacteristics;

   public GameObject test;


   private InputDevice targetDevice;

   void Start()

   {

       controllerCharacteristics = InputDeviceCharacteristics.Right | InputDeviceCharacteristics.Controller;

       var devices = new List<InputDevice>();

       InputDevices.GetDevicesWithCharacteristics(controllerCharacteristics, devices);

       if (devices.Count > 0){

           targetDevice = devices[0];

       }

   }


   private void Update()

   {

       // Start by figuring out which hand we're tracking

       XRNode node = _hand == Hand.LeftHand ? XRNode.LeftHand : XRNode.RightHand;

       string trigger = _hand == Hand.LeftHand ? "Left Trigger" : "Right Trigger";


       // Get the position & rotation of the hand

       bool handIsTracking = UpdatePose(node, ref _handPosition, ref _handRotation);


       // Figure out if the trigger is pressed or not

       bool triggerPressed = targetDevice.TryGetFeatureValue(CommonUsages.trigger, out float triggerValue) && triggerValue > 0.1f;


       // If we lose tracking, stop drawing

       if (!handIsTracking)

           triggerPressed = false;


       // If the trigger is pressed and we haven't created a new brush stroke to draw, create one!

       if (triggerPressed && _activeBrushStroke == null)

       {

           // Instantiate a copy of the Brush Stroke prefab.

           GameObject brushStrokeGameObject = Instantiate(_brushStrokePrefab);


           // Grab the BrushStroke component from it

           _activeBrushStroke = brushStrokeGameObject.GetComponent<BrushStroke>();


           // Tell the BrushStroke to begin drawing at the current brush position

           _activeBrushStroke.BeginBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);

       }


       // If the trigger is pressed, and we have a brush stroke, move the brush stroke to the new brush tip position

       if (triggerPressed)

           _activeBrushStroke.MoveBrushTipToPoint(_handPosition, _handRotation);


       // If the trigger is no longer pressed, and we still have an active brush stroke, mark it as finished and clear it.

       if (!triggerPressed && _activeBrushStroke != null)

       {

           _activeBrushStroke.EndBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);

           _activeBrushStroke = null;

       }

   }


   //// Utility


   // Given an XRNode, get the current position & rotation. If it's not tracking, don't modify the position & rotation.

   private static bool UpdatePose(XRNode node, ref Vector3 position, ref Quaternion rotation)

   {

       List<XRNodeState> nodeStates = new List<XRNodeState>();

       InputTracking.GetNodeStates(nodeStates);


       foreach (XRNodeState nodeState in nodeStates)

       {

           if (nodeState.nodeType == XRNode.RightHand)

           {

               Vector3 nodePosition;

               Quaternion nodeRotation;

               bool gotPosition = nodeState.TryGetPosition(out nodePosition);

               bool gotRotation = nodeState.TryGetRotation(out nodeRotation);


               if (gotPosition)

                   position = nodePosition;

               if (gotRotation)

                   rotation = nodeRotation;

               return gotPosition;

           }

       }

       return false;

   }

}


Step 5. Test on your VR! When you press your right trigger, a brush stroke with the color of your choice should now appear. If it doesn't, make sure that you are on Unity version 2019 and that you have a device-based XR Origin