Creating Unity VR Drawing App

by Jennifer Wang (2022)

Requirements

Installation of Unity (2019)
Follow the Unity XR Setup Guide (use a device-based XR Origin instead)

Step 1. Create an empty object called Mesh and add the components Mesh Filter and Mesh Renderer (select a material for element 0 as the color of the brush stroke). Add the script "Mesh" with the following code as a component to this object.

using System.Collections;

using System.Collections.Generic;

using UnityEngine;

public class BrushStrokeMesh : MonoBehaviour {

[SerializeField]

private float _brushStrokeWidth = 0.05f;

private Mesh _mesh;

private List<Vector3> _vertices;

private List<Vector3> _normals;

private bool _skipLastRibbonPoint;

public bool skipLastRibbonPoint { get { return _skipLastRibbonPoint; } set { if (value == _skipLastRibbonPoint) return; _skipLastRibbonPoint = value; UpdateGeometry(); } }

private void Awake() {

MeshFilter filter = gameObject.GetComponent<MeshFilter>();

_mesh = filter.mesh;

_vertices = new List<Vector3>();

_normals = new List<Vector3>();

// In addition to clearing the ribbon, this adds a ribbon point that we'll move each frame to match the

// brush tip position so that the brush stroke appears to paint continuously.

ClearRibbon();

}

// Insert a new ribbon point just before the final ribbon point

public void InsertRibbonPoint(Vector3 position, Quaternion rotation) {

// Calculate vertices + normal for ribbon point

Vector3 p1;

Vector3 p2;

Vector3 normal;

CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);

// Insert into vertices array

_vertices.Insert(_vertices.Count-4, p1);

_vertices.Insert(_vertices.Count-4, p2);

_vertices.Insert(_vertices.Count-4, p1);

_vertices.Insert(_vertices.Count-4, p2);

// Insert into normals array

_normals.Insert(_normals.Count-4, normal);

_normals.Insert(_normals.Count-4, -normal);

// Update the mesh

UpdateGeometry();

}

public void UpdateLastRibbonPoint(Vector3 position, Quaternion rotation) {

// Calculate vertices + normal for ribbon point

Vector3 p1;

Vector3 p2;

Vector3 normal;

CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);

int lastIndex = _vertices.Count-4;

// Update vertices

_vertices[lastIndex] = p1;

_vertices[lastIndex+1] = p2;

_vertices[lastIndex+2] = p1;

_vertices[lastIndex+3] = p2;

// Update normals

_normals[lastIndex] = normal;

_normals[lastIndex+1] = normal;

_normals[lastIndex+2] = -normal;

_normals[lastIndex+3] = -normal;

// Update the mesh

UpdateGeometry();

}

public void ClearRibbon() {

// Clear vertices & normals

_vertices.Clear();

_normals.Clear();

// Create last ribbon point

_vertices.Add(Vector3.zero);

_normals.Add(Vector3.zero);

// Update the mesh

UpdateGeometry();

}

private void CalculateVerticesAndNormalForRibbonPoint(Vector3 position, Quaternion rotation, float width, out Vector3 p1, out Vector3 p2, out Vector3 normal) {

p1 = position + rotation * new Vector3(-width/2.0f, 0.0f, 0.0f);

p2 = position + rotation * new Vector3( width/2.0f, 0.0f, 0.0f);

normal = rotation * Vector3.up;

}

private void UpdateGeometry() {

int numberOfVertices = _vertices.Count;

if (skipLastRibbonPoint)

numberOfVertices -= 4;

if (numberOfVertices < 8) {

_mesh.vertices = new Vector3[0];

_mesh.normals = new Vector3[0];

_mesh.triangles = new int[0];

_mesh.RecalculateBounds();

return;

}

// Would probably make sense to just generate the new triangles rather than regenerating all of them all of the time.

int numberOfSegments = numberOfVertices/4 - 1;

int numberOfTriangles = numberOfSegments * 4; // Two on the front side, two on the back.

int[] triangles = new int[numberOfTriangles*3];

for (int i = 0; i < numberOfSegments; i++) {

// Front

int p1 = i*4;

int p2 = i*4+1;

int p3 = i*4+4;

int p4 = i*4+5;

// Back

int p1b = i*4+2;

int p2b = i*4+3;

int p3b = i*4+6;

int p4b = i*4+7;

// Front

triangles[i*12] = p1;

triangles[i*12+1] = p2;

triangles[i*12+2] = p3;

triangles[i*12+3] = p2;

triangles[i*12+4] = p4;

triangles[i*12+5] = p3;

// Back

triangles[i*12+6] = p1b;

triangles[i*12+7] = p3b;

triangles[i*12+8] = p2b;

triangles[i*12+9] = p2b;

triangles[i*12+10] = p3b;

triangles[i*12+11] = p4b;

}

_mesh.vertices = _vertices.ToArray();

_mesh.normals = _normals.ToArray();

_mesh.triangles = triangles;

_mesh.RecalculateBounds();

}

Step 2. Create an empty object called BrushStroke and add the script "BrushStroke" with the following code as a component to this object. Then add Mesh as a child to this object.

using System.Collections.Generic;

using UnityEngine;

public class BrushStroke : MonoBehaviour {

[SerializeField]

private BrushStrokeMesh _mesh = null;

// Ribbon State

struct RibbonPoint {

public Vector3 position;

public Quaternion rotation;

}

private List<RibbonPoint> _ribbonPoints = new List<RibbonPoint>();

private Vector3 _brushTipPosition;

private Quaternion _brushTipRotation;

private bool _brushStrokeFinalized;

// Smoothing

private Vector3 _ribbonEndPosition;

private Quaternion _ribbonEndRotation = Quaternion.identity;

// Mesh

private Vector3 _previousRibbonPointPosition;

private Quaternion _previousRibbonPointRotation = Quaternion.identity;

// Unity Events

private void Update() {

// Animate the end of the ribbon towards the brush tip

AnimateLastRibbonPointTowardsBrushTipPosition();

// Add a ribbon segment if the end of the ribbon has moved far enough

AddRibbonPointIfNeeded();

}

// Interface

public void BeginBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {

// Update the model

_brushTipPosition = position;

_brushTipRotation = rotation;

// Update last ribbon point to match brush tip position & rotation

_ribbonEndPosition = position;

_ribbonEndRotation = rotation;

_mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);

}

public void MoveBrushTipToPoint(Vector3 position, Quaternion rotation) {

_brushTipPosition = position;

_brushTipRotation = rotation;

}

public void EndBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {

// Add a final ribbon point and mark the stroke as finalized

AddRibbonPoint(position, rotation);

_brushStrokeFinalized = true;

}

// Ribbon drawing

private void AddRibbonPointIfNeeded() {

// If the brush stroke is finalized, stop trying to add points to it.

if (_brushStrokeFinalized)

return;

if (Vector3.Distance(_ribbonEndPosition, _previousRibbonPointPosition) >= 0.01f ||

Quaternion.Angle(_ribbonEndRotation, _previousRibbonPointRotation) >= 10.0f) {

// Add ribbon point model to ribbon points array. This will fire the RibbonPointAdded event to update the mesh.

AddRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);

// Store the ribbon point position & rotation for the next time we do this calculation

_previousRibbonPointPosition = _ribbonEndPosition;

_previousRibbonPointRotation = _ribbonEndRotation;

}

private void AddRibbonPoint(Vector3 position, Quaternion rotation) {

// Create the ribbon point

RibbonPoint ribbonPoint = new RibbonPoint();

ribbonPoint.position = position;

ribbonPoint.rotation = rotation;

_ribbonPoints.Add(ribbonPoint);

// Update the mesh

_mesh.InsertRibbonPoint(position, rotation);

}

// Brush tip + smoothing

private void AnimateLastRibbonPointTowardsBrushTipPosition() {

// If the brush stroke is finalized, skip the brush tip mesh, and stop animating the brush tip.

if (_brushStrokeFinalized) {

_mesh.skipLastRibbonPoint = true;

return;

}

Vector3 brushTipPosition = _brushTipPosition;

Quaternion brushTipRotation = _brushTipRotation;

// If the end of the ribbon has reached the brush tip position, we can bail early.

if (Vector3.Distance(_ribbonEndPosition, brushTipPosition) <= 0.0001f &&

Quaternion.Angle(_ribbonEndRotation, brushTipRotation) <= 0.01f) {

return;

}

// Move the end of the ribbon towards the brush tip position

_ribbonEndPosition = Vector3.Lerp(_ribbonEndPosition, brushTipPosition, 25.0f * Time.deltaTime);

_ribbonEndRotation = Quaternion.Slerp(_ribbonEndRotation, brushTipRotation, 25.0f * Time.deltaTime);

// Update the end of the ribbon mesh

_mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);

}

Step 3. Drag the object BrushStroke into assets to set it to a prefab. Delete the object from scene.

Step 4. Create an empty object called Brush and add the following script to its components.

using System.Collections.Generic;

using UnityEngine;

using UnityEngine.XR;

using UnityEngine.UI;

public class Brush : MonoBehaviour

{

// Prefab to instantiate when we draw a new brush stroke

[SerializeField] private GameObject _brushStrokePrefab = null;

// Which hand should this brush instance track?

private enum Hand { LeftHand, RightHand };

[SerializeField] private Hand _hand = Hand.RightHand;

// Used to keep track of the current brush tip position and the actively drawing brush stroke

private Vector3 _handPosition;

private Quaternion _handRotation;

private BrushStroke _activeBrushStroke;

public InputDeviceCharacteristics controllerCharacteristics;

public GameObject test;

private InputDevice targetDevice;

void Start()

{

controllerCharacteristics = InputDeviceCharacteristics.Right | InputDeviceCharacteristics.Controller;

var devices = new List<InputDevice>();

InputDevices.GetDevicesWithCharacteristics(controllerCharacteristics, devices);

if (devices.Count > 0){

targetDevice = devices[0];

}

private void Update()

{

// Start by figuring out which hand we're tracking

XRNode node = _hand == Hand.LeftHand ? XRNode.LeftHand : XRNode.RightHand;

string trigger = _hand == Hand.LeftHand ? "Left Trigger" : "Right Trigger";

// Get the position & rotation of the hand

bool handIsTracking = UpdatePose(node, ref _handPosition, ref _handRotation);

// Figure out if the trigger is pressed or not

bool triggerPressed = targetDevice.TryGetFeatureValue(CommonUsages.trigger, out float triggerValue) && triggerValue > 0.1f;

// If we lose tracking, stop drawing

if (!handIsTracking)

triggerPressed = false;

// If the trigger is pressed and we haven't created a new brush stroke to draw, create one!

if (triggerPressed && _activeBrushStroke == null)

{

// Instantiate a copy of the Brush Stroke prefab.

GameObject brushStrokeGameObject = Instantiate(_brushStrokePrefab);

// Grab the BrushStroke component from it

_activeBrushStroke = brushStrokeGameObject.GetComponent<BrushStroke>();

// Tell the BrushStroke to begin drawing at the current brush position

_activeBrushStroke.BeginBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);

}

// If the trigger is pressed, and we have a brush stroke, move the brush stroke to the new brush tip position

if (triggerPressed)

_activeBrushStroke.MoveBrushTipToPoint(_handPosition, _handRotation);

// If the trigger is no longer pressed, and we still have an active brush stroke, mark it as finished and clear it.

if (!triggerPressed && _activeBrushStroke != null)

{

_activeBrushStroke.EndBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);

_activeBrushStroke = null;

}

//// Utility

// Given an XRNode, get the current position & rotation. If it's not tracking, don't modify the position & rotation.

private static bool UpdatePose(XRNode node, ref Vector3 position, ref Quaternion rotation)

{

List<XRNodeState> nodeStates = new List<XRNodeState>();

InputTracking.GetNodeStates(nodeStates);

foreach (XRNodeState nodeState in nodeStates)

{

if (nodeState.nodeType == XRNode.RightHand)

{

Vector3 nodePosition;

Quaternion nodeRotation;

bool gotPosition = nodeState.TryGetPosition(out nodePosition);

bool gotRotation = nodeState.TryGetRotation(out nodeRotation);

if (gotPosition)

position = nodePosition;

if (gotRotation)

rotation = nodeRotation;

return gotPosition;

}

return false;

}

Step 5. Test on your VR! When you press your right trigger, a brush stroke with the color of your choice should now appear. If it doesn't, make sure that you are on Unity version 2019 and that you have a device-based XR Origin