fix(nn/rnn): corrected LSTM and GRU binding function. feat(libtch/README): updated with function created multiple ctensor in C land memory
This commit is contained in:
sugarme
parent
9817f7393a
commit
42c02b0f65
43
example/rnn/main.go
Normal file
43
example/rnn/main.go
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/sugarme/gotch"
|
||||
"github.com/sugarme/gotch/nn"
|
||||
ts "github.com/sugarme/gotch/tensor"
|
||||
)
|
||||
|
||||
func rnnTest(rnnConfig nn.RNNConfig) {
|
||||
|
||||
var (
|
||||
batchDim int64 = 5
|
||||
// seqLen int64 = 3
|
||||
inputDim int64 = 2
|
||||
outputDim int64 = 4
|
||||
)
|
||||
|
||||
vs := nn.NewVarStore(gotch.CPU)
|
||||
path := vs.Root()
|
||||
|
||||
gru := nn.NewGRU(&path, inputDim, outputDim, rnnConfig)
|
||||
|
||||
numDirections := int64(1)
|
||||
if rnnConfig.Bidirectional {
|
||||
numDirections = 2
|
||||
}
|
||||
layerDim := rnnConfig.NumLayers * numDirections
|
||||
|
||||
// Step test
|
||||
input := ts.MustRandn([]int64{batchDim, inputDim}, gotch.Float, gotch.CPU)
|
||||
output := gru.Step(input, gru.ZeroState(batchDim).(nn.GRUState))
|
||||
|
||||
fmt.Printf("Expected ouput shape: %v\n", []int64{layerDim, batchDim, outputDim})
|
||||
fmt.Printf("Got output shape: %v\n", output.(nn.GRUState).Tensor.MustSize())
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
rnnTest(nn.DefaultRNNConfig())
|
||||
}
|
||||
|
|
@ -110,3 +110,89 @@ func GetAndResetLastErr() *C.char{
|
|||
```
|
||||
|
||||
|
||||
## Multiple Tensors Created In C Land Memory
|
||||
|
||||
- When there are multiple Ctensor created in C land memory. A first Ctensor
|
||||
pointer will be created and given to the C function. It will create
|
||||
consecutive Ctensor(s) based on this pointer. The next pointer(s) can be
|
||||
calulated based on this pointer and its size.
|
||||
|
||||
- Example: **lstm* function
|
||||
|
||||
+ **C function**
|
||||
|
||||
```C
|
||||
void atg_lstm(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first);
|
||||
```
|
||||
|
||||
+ **Go wrapper function**
|
||||
|
||||
```go
|
||||
func AtgLstm(ptr *Ctensor, input Ctensor, hxData []Ctensor, hxLen int, paramsData []Ctensor, paramsLen int, hasBiases int, numLayers int64, dropout float64, train int, bidirectional int, batchFirst int) {
|
||||
|
||||
chxDataPtr := (*Ctensor)(unsafe.Pointer(&hxData[0]))
|
||||
chxLen := *(*C.int)(unsafe.Pointer(&hxLen))
|
||||
cparamsDataPtr := (*Ctensor)(unsafe.Pointer(¶msData[0]))
|
||||
cparamsLen := *(*C.int)(unsafe.Pointer(¶msLen))
|
||||
chasBiases := *(*C.int)(unsafe.Pointer(&hasBiases))
|
||||
cnumLayers := *(*C.int64_t)(unsafe.Pointer(&numLayers))
|
||||
cdropout := *(*C.double)(unsafe.Pointer(&dropout))
|
||||
ctrain := *(*C.int)(unsafe.Pointer(&train))
|
||||
cbidirectional := *(*C.int)(unsafe.Pointer(&bidirectional))
|
||||
cbatchFirst := *(*C.int)(unsafe.Pointer(&batchFirst))
|
||||
|
||||
C.atg_lstm(ptr, input, chxDataPtr, chxLen, cparamsDataPtr, cparamsLen, chasBiases, cnumLayers, cdropout, ctrain, cbidirectional, cbatchFirst)
|
||||
}
|
||||
```
|
||||
|
||||
+ **Go API function**
|
||||
|
||||
```go
|
||||
func (ts Tensor) LSTM(hxData []Tensor, paramsData []Tensor, hasBiases bool, numLayers int64, dropout float64, train bool, bidirectional bool, batchFirst bool) (output, h, c Tensor, err error) {
|
||||
|
||||
// NOTE: `atg_lstm` will create 3 consecutive Ctensors in memory of C land. The first
|
||||
// Ctensor will have address given by `ctensorPtr1` here.
|
||||
// The next pointers can be calculated based on `ctensorPtr1`
|
||||
ctensorPtr1 := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ctensorPtr2 := (*lib.Ctensor)(unsafe.Pointer(uintptr(unsafe.Pointer(ctensorPtr1)) + unsafe.Sizeof(ctensorPtr1)))
|
||||
ctensorPtr3 := (*lib.Ctensor)(unsafe.Pointer(uintptr(unsafe.Pointer(ctensorPtr2)) + unsafe.Sizeof(ctensorPtr1)))
|
||||
|
||||
var chxData []lib.Ctensor
|
||||
for _, t := range hxData {
|
||||
chxData = append(chxData, t.ctensor)
|
||||
}
|
||||
|
||||
var cparamsData []lib.Ctensor
|
||||
for _, t := range paramsData {
|
||||
cparamsData = append(cparamsData, t.ctensor)
|
||||
}
|
||||
|
||||
chasBiases := 0
|
||||
if hasBiases {
|
||||
chasBiases = 1
|
||||
}
|
||||
ctrain := 0
|
||||
if train {
|
||||
ctrain = 1
|
||||
}
|
||||
cbidirectional := 0
|
||||
if bidirectional {
|
||||
cbidirectional = 1
|
||||
}
|
||||
cbatchFirst := 0
|
||||
if batchFirst {
|
||||
cbatchFirst = 1
|
||||
}
|
||||
|
||||
lib.AtgLstm(ctensorPtr1, ts.ctensor, chxData, len(hxData), cparamsData, len(paramsData), chasBiases, numLayers, dropout, ctrain, cbidirectional, cbatchFirst)
|
||||
err = TorchErr()
|
||||
if err != nil {
|
||||
return output, h, c, err
|
||||
}
|
||||
|
||||
return Tensor{ctensor: *ctensorPtr1}, Tensor{ctensor: *ctensorPtr2}, Tensor{ctensor: *ctensorPtr3}, nil
|
||||
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -434,7 +434,7 @@ func AtgConvTranspose3d(ptr *Ctensor, input Ctensor, weight Ctensor, bias Ctenso
|
|||
}
|
||||
|
||||
// void atg_lstm(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first);
|
||||
func AtgLstm(ctensorsPtr []*Ctensor, input Ctensor, hxData []Ctensor, hxLen int, paramsData []Ctensor, paramsLen int, hasBiases int, numLayers int64, dropout float64, train int, bidirectional int, batchFirst int) {
|
||||
func AtgLstm(ptr *Ctensor, input Ctensor, hxData []Ctensor, hxLen int, paramsData []Ctensor, paramsLen int, hasBiases int, numLayers int64, dropout float64, train int, bidirectional int, batchFirst int) {
|
||||
|
||||
chxDataPtr := (*Ctensor)(unsafe.Pointer(&hxData[0]))
|
||||
chxLen := *(*C.int)(unsafe.Pointer(&hxLen))
|
||||
|
|
@ -447,11 +447,11 @@ func AtgLstm(ctensorsPtr []*Ctensor, input Ctensor, hxData []Ctensor, hxLen int,
|
|||
cbidirectional := *(*C.int)(unsafe.Pointer(&bidirectional))
|
||||
cbatchFirst := *(*C.int)(unsafe.Pointer(&batchFirst))
|
||||
|
||||
C.atg_lstm(ctensorsPtr[0], input, chxDataPtr, chxLen, cparamsDataPtr, cparamsLen, chasBiases, cnumLayers, cdropout, ctrain, cbidirectional, cbatchFirst)
|
||||
C.atg_lstm(ptr, input, chxDataPtr, chxLen, cparamsDataPtr, cparamsLen, chasBiases, cnumLayers, cdropout, ctrain, cbidirectional, cbatchFirst)
|
||||
}
|
||||
|
||||
// void atg_gru(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first);
|
||||
func AtgGru(ctensorsPtr []*Ctensor, input Ctensor, hx Ctensor, paramsData []Ctensor, paramsLen int, hasBiases int, numLayers int64, dropout float64, train int, bidirectional int, batchFirst int) {
|
||||
func AtgGru(ptr *Ctensor, input Ctensor, hx Ctensor, paramsData []Ctensor, paramsLen int, hasBiases int, numLayers int64, dropout float64, train int, bidirectional int, batchFirst int) {
|
||||
|
||||
cparamsDataPtr := (*Ctensor)(unsafe.Pointer(¶msData[0]))
|
||||
cparamsLen := *(*C.int)(unsafe.Pointer(¶msLen))
|
||||
|
|
@ -462,5 +462,16 @@ func AtgGru(ctensorsPtr []*Ctensor, input Ctensor, hx Ctensor, paramsData []Cten
|
|||
cbidirectional := *(*C.int)(unsafe.Pointer(&bidirectional))
|
||||
cbatchFirst := *(*C.int)(unsafe.Pointer(&batchFirst))
|
||||
|
||||
C.atg_gru(ctensorsPtr[0], input, hx, cparamsDataPtr, cparamsLen, chasBiases, cnumLayers, cdropout, ctrain, cbidirectional, cbatchFirst)
|
||||
C.atg_gru(ptr, input, hx, cparamsDataPtr, cparamsLen, chasBiases, cnumLayers, cdropout, ctrain, cbidirectional, cbatchFirst)
|
||||
}
|
||||
|
||||
// void atg_randn(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device);
|
||||
func AtgRandn(ptr *Ctensor, sizeData []int64, sizeLen int, optionsKind int32, optionsDevice int32) {
|
||||
|
||||
csizeDataPtr := (*C.int64_t)(unsafe.Pointer(&sizeData[0]))
|
||||
csizeLen := *(*C.int)(unsafe.Pointer(&sizeLen))
|
||||
coptionKind := *(*C.int)(unsafe.Pointer(&optionsKind))
|
||||
coptionDevice := *(*C.int)(unsafe.Pointer(&optionsDevice))
|
||||
|
||||
C.atg_randn(ptr, csizeDataPtr, csizeLen, coptionKind, coptionDevice)
|
||||
}
|
||||
|
|
|
|||
28
nn/rnn.go
28
nn/rnn.go
|
|
@ -139,10 +139,14 @@ func (l LSTM) ZeroState(batchDim int64) (retVal State) {
|
|||
}
|
||||
}
|
||||
|
||||
func (l LSTM) Step(input ts.Tensor, inState State) (retVal State) {
|
||||
func (l LSTM) Step(input ts.Tensor, inState LSTMState) (retVal State) {
|
||||
ip := input.MustUnsqueeze(1, false)
|
||||
|
||||
_, state := l.SeqInit(ip, inState.(LSTMState))
|
||||
output, state := l.SeqInit(ip, inState)
|
||||
|
||||
// NOTE: though we won't use `output`, it is a Ctensor created in C land, so
|
||||
// it should be cleaned up here to prevent memory hold-up.
|
||||
output.MustDrop()
|
||||
|
||||
return state
|
||||
}
|
||||
|
|
@ -151,9 +155,9 @@ func (l LSTM) Seq(input ts.Tensor) (ts.Tensor, State) {
|
|||
return defaultSeq(l, input)
|
||||
}
|
||||
|
||||
func (l LSTM) SeqInit(input ts.Tensor, inState State) (ts.Tensor, State) {
|
||||
func (l LSTM) SeqInit(input ts.Tensor, inState LSTMState) (ts.Tensor, State) {
|
||||
|
||||
output, h, c := input.MustLSTM([]ts.Tensor{inState.(LSTMState).Tensor1, inState.(LSTMState).Tensor2}, l.flatWeights, l.config.HasBiases, l.config.NumLayers, l.config.Dropout, l.config.Train, l.config.Bidirectional, l.config.BatchFirst)
|
||||
output, h, c := input.MustLSTM([]ts.Tensor{inState.Tensor1, inState.Tensor2}, l.flatWeights, l.config.HasBiases, l.config.NumLayers, l.config.Dropout, l.config.Train, l.config.Bidirectional, l.config.BatchFirst)
|
||||
|
||||
return output, LSTMState{
|
||||
Tensor1: h,
|
||||
|
|
@ -225,13 +229,19 @@ func (g GRU) ZeroState(batchDim int64) (retVal State) {
|
|||
layerDim := g.config.NumLayers * numDirections
|
||||
shape := []int64{layerDim, batchDim, g.hiddenDim}
|
||||
|
||||
return ts.MustZeros(shape, gotch.Float.CInt(), g.device.CInt())
|
||||
tensor := ts.MustZeros(shape, gotch.Float.CInt(), g.device.CInt())
|
||||
|
||||
return GRUState{Tensor: tensor}
|
||||
}
|
||||
|
||||
func (g GRU) Step(input ts.Tensor, inState State) (retVal State) {
|
||||
func (g GRU) Step(input ts.Tensor, inState GRUState) (retVal State) {
|
||||
ip := input.MustUnsqueeze(1, false)
|
||||
|
||||
_, state := g.SeqInit(ip, inState.(LSTMState))
|
||||
output, state := g.SeqInit(ip, inState)
|
||||
|
||||
// NOTE: though we won't use `output`, it is a Ctensor created in C land, so
|
||||
// it should be cleaned up here to prevent memory hold-up.
|
||||
output.MustDrop()
|
||||
|
||||
return state
|
||||
}
|
||||
|
|
@ -240,9 +250,9 @@ func (g GRU) Seq(input ts.Tensor) (ts.Tensor, State) {
|
|||
return defaultSeq(g, input)
|
||||
}
|
||||
|
||||
func (g GRU) SeqInit(input ts.Tensor, inState State) (ts.Tensor, State) {
|
||||
func (g GRU) SeqInit(input ts.Tensor, inState GRUState) (ts.Tensor, State) {
|
||||
|
||||
output, h := input.MustGRU(inState.(GRUState).Tensor, g.flatWeights, g.config.HasBiases, g.config.NumLayers, g.config.Dropout, g.config.Train, g.config.Bidirectional, g.config.BatchFirst)
|
||||
output, h := input.MustGRU(inState.Tensor, g.flatWeights, g.config.HasBiases, g.config.NumLayers, g.config.Dropout, g.config.Train, g.config.Bidirectional, g.config.BatchFirst)
|
||||
|
||||
return output, GRUState{Tensor: h}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1263,12 +1263,12 @@ func MustConvTranspose3D(input, weight, bias Tensor, stride, padding, outputPadd
|
|||
|
||||
func (ts Tensor) LSTM(hxData []Tensor, paramsData []Tensor, hasBiases bool, numLayers int64, dropout float64, train bool, bidirectional bool, batchFirst bool) (output, h, c Tensor, err error) {
|
||||
|
||||
// NOTE: atg_lstm will return an array of 3 Ctensors
|
||||
ts1Ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ts2Ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ts3Ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
var ctensorsPtr []*lib.Ctensor
|
||||
ctensorsPtr = append(ctensorsPtr, ts1Ptr, ts2Ptr, ts3Ptr)
|
||||
// NOTE: `atg_lstm` will create 3 consecutive Ctensors in memory of C land. The first
|
||||
// Ctensor will have address given by `ctensorPtr1` here.
|
||||
// The next pointers can be calculated based on `ctensorPtr1`
|
||||
ctensorPtr1 := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ctensorPtr2 := (*lib.Ctensor)(unsafe.Pointer(uintptr(unsafe.Pointer(ctensorPtr1)) + unsafe.Sizeof(ctensorPtr1)))
|
||||
ctensorPtr3 := (*lib.Ctensor)(unsafe.Pointer(uintptr(unsafe.Pointer(ctensorPtr2)) + unsafe.Sizeof(ctensorPtr1)))
|
||||
|
||||
var chxData []lib.Ctensor
|
||||
for _, t := range hxData {
|
||||
|
|
@ -1297,13 +1297,13 @@ func (ts Tensor) LSTM(hxData []Tensor, paramsData []Tensor, hasBiases bool, numL
|
|||
cbatchFirst = 1
|
||||
}
|
||||
|
||||
lib.AtgLstm(ctensorsPtr, ts.ctensor, chxData, len(hxData), cparamsData, len(paramsData), chasBiases, numLayers, dropout, ctrain, cbidirectional, cbatchFirst)
|
||||
lib.AtgLstm(ctensorPtr1, ts.ctensor, chxData, len(hxData), cparamsData, len(paramsData), chasBiases, numLayers, dropout, ctrain, cbidirectional, cbatchFirst)
|
||||
err = TorchErr()
|
||||
if err != nil {
|
||||
return output, h, c, err
|
||||
}
|
||||
|
||||
return Tensor{ctensor: *ts1Ptr}, Tensor{ctensor: *ts2Ptr}, Tensor{ctensor: *ts3Ptr}, nil
|
||||
return Tensor{ctensor: *ctensorPtr1}, Tensor{ctensor: *ctensorPtr2}, Tensor{ctensor: *ctensorPtr3}, nil
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -1319,11 +1319,11 @@ func (ts Tensor) MustLSTM(hxData []Tensor, paramsData []Tensor, hasBiases bool,
|
|||
|
||||
func (ts Tensor) GRU(hx Tensor, paramsData []Tensor, hasBiases bool, numLayers int64, dropout float64, train bool, bidirectional bool, batchFirst bool) (output, h Tensor, err error) {
|
||||
|
||||
// NOTE: atg_gru will returns an array of 2 Ctensor
|
||||
ts1Ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ts2Ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
var ctensorsPtr []*lib.Ctensor
|
||||
ctensorsPtr = append(ctensorsPtr, ts1Ptr, ts2Ptr)
|
||||
// NOTE: `atg_gru` will create 2 consecutive Ctensors in memory of C land.
|
||||
// The first Ctensor will have address given by `ctensorPtr1` here.
|
||||
// The next pointer can be calculated based on `ctensorPtr1`
|
||||
ctensorPtr1 := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
ctensorPtr2 := (*lib.Ctensor)(unsafe.Pointer(uintptr(unsafe.Pointer(ctensorPtr1)) + unsafe.Sizeof(ctensorPtr1)))
|
||||
|
||||
var cparamsData []lib.Ctensor
|
||||
for _, t := range paramsData {
|
||||
|
|
@ -1347,13 +1347,14 @@ func (ts Tensor) GRU(hx Tensor, paramsData []Tensor, hasBiases bool, numLayers i
|
|||
cbatchFirst = 1
|
||||
}
|
||||
|
||||
lib.AtgGru(ctensorsPtr, ts.ctensor, hx.ctensor, cparamsData, len(paramsData), chasBiases, numLayers, dropout, ctrain, cbidirectional, cbatchFirst)
|
||||
lib.AtgGru(ctensorPtr1, ts.ctensor, hx.ctensor, cparamsData, len(paramsData), chasBiases, numLayers, dropout, ctrain, cbidirectional, cbatchFirst)
|
||||
err = TorchErr()
|
||||
if err != nil {
|
||||
return output, h, err
|
||||
}
|
||||
|
||||
return Tensor{ctensor: *ts1Ptr}, Tensor{ctensor: *ts2Ptr}, nil
|
||||
return Tensor{ctensor: *ctensorPtr1}, Tensor{ctensor: *ctensorPtr2}, nil
|
||||
|
||||
}
|
||||
|
||||
func (ts Tensor) MustGRU(hx Tensor, paramsData []Tensor, hasBiases bool, numLayers int64, dropout float64, train bool, bidirectional bool, batchFirst bool) (output, h Tensor) {
|
||||
|
|
@ -1364,3 +1365,28 @@ func (ts Tensor) MustGRU(hx Tensor, paramsData []Tensor, hasBiases bool, numLaye
|
|||
|
||||
return output, h
|
||||
}
|
||||
|
||||
func Randn(sizeData []int64, optionsKind gotch.DType, optionsDevice gotch.Device) (retVal Tensor, err error) {
|
||||
|
||||
ptr := (*lib.Ctensor)(unsafe.Pointer(C.malloc(0)))
|
||||
|
||||
lib.AtgRandn(ptr, sizeData, len(sizeData), optionsKind.CInt(), optionsDevice.CInt())
|
||||
err = TorchErr()
|
||||
if err != nil {
|
||||
return retVal, err
|
||||
}
|
||||
|
||||
retVal = Tensor{ctensor: *ptr}
|
||||
|
||||
return retVal, nil
|
||||
}
|
||||
|
||||
func MustRandn(sizeData []int64, optionsKind gotch.DType, optionsDevice gotch.Device) (retVal Tensor) {
|
||||
|
||||
retVal, err := Randn(sizeData, optionsKind, optionsDevice)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
return retVal
|
||||
}
|
||||
|
|
|
|||
|
|
@ -993,5 +993,7 @@ func (r Reduction) ToInt() (retVal int) {
|
|||
func (ts Tensor) Values() []float64 {
|
||||
clone := ts.MustShallowClone()
|
||||
clone.Detach_()
|
||||
// NOTE: this for 2D tensor.
|
||||
// TODO: flatten nd tensor to slice
|
||||
return []float64{clone.MustView([]int64{-1}, true).MustFloat64Value([]int64{-1})}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user