How do you set a value to a pointer containing nil using reflection

I'm trying to set a value to a nil pointer in a struct like so.

// http://ift.tt/2uyxGfK
package main

import (
    "fmt"
    "reflect"
)

type T struct {
    A *int
}

func main() {
    fmt.Println("Hello, playground")
    t := &T{}

    v := 1
    vptr := &v

    CopyValue(vptr, t.A) // I want to set t.A to contain 1
}

func CopyValue(src interface{}, dest interface{}) {
    srcRef := reflect.ValueOf(src)
    if srcRef.Kind() == reflect.Ptr {
        srcRef = srcRef.Elem()
    }

    destRef := reflect.New(srcRef.Type()).Elem()
    destRef.Set(srcRef)
    reflect.ValueOf(dest).Elem().Set(destRef)
}

However, I encounter the following error:

panic: reflect: call of reflect.Value.Set on zero Value

goroutine 1 [running]:
reflect.flag.mustBeAssignable(0x0, 0x1040a128)
    /usr/local/go/src/reflect/value.go:221 +0x260
reflect.Value.Set(0x0, 0x0, 0x0, 0xdefc0, 0x1040a128, 0x182)
    /usr/local/go/src/reflect/value.go:1339 +0x40
main.CopyValue(0xd7860, 0x1040a124, 0xd7860, 0x0)
    /tmp/sandbox487854080/main.go:30 +0x1a0
main.main()
    /tmp/sandbox487854080/main.go:19 +0x100

What am I doing wrong?